From d5fc5305705bd5a308d1c5cab8da0fba1b5d0320 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Tue, 25 May 2010 12:46:00 -0400 Subject: add fabric app, begin moving relevant parts of showroom over to it --- ebin/fabric.app | 15 +++ src/fabric.erl | 6 ++ src/fabric_api.erl | 262 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/fabric_create.erl | 141 +++++++++++++++++++++++++++ 4 files changed, 424 insertions(+) create mode 100644 ebin/fabric.app create mode 100644 src/fabric.erl create mode 100644 src/fabric_api.erl create mode 100644 src/fabric_create.erl diff --git a/ebin/fabric.app b/ebin/fabric.app new file mode 100644 index 00000000..76de79ff --- /dev/null +++ b/ebin/fabric.app @@ -0,0 +1,15 @@ +%% fabric app resource file + +{application, fabric, + [{description, "clustered couchdb functions"}, + {vsn, "0.1.0"}, + {modules, [ + fabric, + fabric_api, + fabric_create + ]}, + {registered, []}, + {included_applications, []}, + {applications, [kernel, stdlib, couch, rexi, dynomite]}, + {start_phases, []} + ]}. diff --git a/src/fabric.erl b/src/fabric.erl new file mode 100644 index 00000000..d588265f --- /dev/null +++ b/src/fabric.erl @@ -0,0 +1,6 @@ +-module(fabric). + +-export([create_db/2]). + +create_db(DbName, Options) -> + fabric_create:create_db(DbName, Options). diff --git a/src/fabric_api.erl b/src/fabric_api.erl new file mode 100644 index 00000000..a26aa7de --- /dev/null +++ b/src/fabric_api.erl @@ -0,0 +1,262 @@ +%% This is a raw Erlang API for CouchDB in a Cloudant cluster +%% It makes use of clustering facilities + +-module(fabric_api). +-author('adam@cloudant.com'). +-author('brad@cloudant.com'). + +-include("../../couch/src/couch_db.hrl"). + +-compile(export_all). + +-type response() :: any(). + +%% dialyzer doesn't have recursive types, so this is necessarily wrong +-type ejson_value() :: true | false | number() | bstring() | list(). +-type ejson() :: {[{bstring(), ejson_value()}]}. + +%% Database + +-spec db_path(bstring(), bstring()) -> bstring(). +db_path(RawUri, Customer) -> + showroom_db:db_path(RawUri, Customer). + +-spec all_databases(string()) -> {ok, [bstring()]}. +all_databases(Customer) -> + showroom_db:all_databases(Customer). + +-spec create_db(bstring(), [any()]) -> {ok, #db{}} | {error, any()}. +create_db(DbName, Options) -> + fabric:create_db(DbName, Options). + +-spec delete_db(bstring(), [any()]) -> ok | not_found | {error, atom()}. +delete_db(DbName, Options) -> + showroom_db:delete_db(DbName, Options). + +-spec open_db(bstring(), [any()]) -> {ok, #db{}} | {error, any()}. +open_db(DbName, Options) -> + showroom_db:open_db(DbName, Options). + +-spec close_db(#db{}) -> ok. +close_db(Db) -> + showroom_db:close_db(Db). + +-spec get_db_info(#db{}, bstring()) -> {ok, [{atom(), any()}]}. +get_db_info(Db, Customer) -> + showroom_db:get_db_info(Db, Customer). + +-spec get_committed_update_seq(#db{}) -> update_seq(). +get_committed_update_seq(_Db) -> +%% couch_db:get_committed_update_seq(Db). + not_implemented. + +-spec get_purge_seq(#db{}) -> update_seq(). +get_purge_seq(_Db) -> +%% couch_db:get_purge_seq(Db). + not_implemented. + +-spec get_update_seq(#db{}) -> update_seq(). +get_update_seq(_Db) -> +%% couch_db:get_update_seq(Db). + not_implemented. + +-spec compact_db(#db{}) -> ok. +compact_db(_Db) -> +%% couch_db:start_compact(Db). + not_implemented. + +-spec replicate_db(ejson(), #user_ctx{}) -> {ok, ejson()}. +replicate_db(PostBody, UserCtx) -> + showroom_rep:replicate(PostBody, UserCtx). + +-spec ensure_full_commit(#db{}) -> {ok, InstanceStartTime::bstring()}. +ensure_full_commit(Db) -> + showroom_db:ensure_full_commit(Db), + {ok, <<"0">>}. + +-spec increment_update_seq(#db{}) -> {ok, update_seq()}. +increment_update_seq(_Db) -> +%% couch_db:increment_update_seq(Db). + not_implemented. + +-spec get_admins(#db{}) -> [any()]. +get_admins(_Db) -> +%% couch_db:get_admins(Db). + not_implemented. + +-spec set_admins(#db{}, [any()]) -> ok. +set_admins(_Db, _Admins) -> +%% couch_db:set_admins(Db, Admins). + not_implemented. + +-spec get_revs_limit(#db{}) -> pos_integer(). +get_revs_limit(_Db) -> +%% couch_db:get_revs_limit(Db). + not_implemented. + +-spec set_revs_limit(#db{}, pos_integer()) -> ok. +set_revs_limit(_Db, _Limit) -> +%% couch_db:set_revs_limit(Db, Limit). + not_implemented. + + +%% Document + +att_receiver(Req, Length) -> + showroom_att:receiver(Req, Length). + +-spec changes_since(#db{}, main_only | all_docs, update_seq(), + fun(([#doc_info{}], Acc) -> {ok, Acc}), Acc) -> {ok, Acc}. +changes_since(_Db, _Style, _StartSeq, _Fun, _Acc0) -> +%% couch_db:changes_since(Db, Style, StartSeq, Fun, Acc0). + not_implemented. + +-spec enum_docs(#db{}, docid(), fwd | rev, fun((#full_doc_info{}, + Offset::integer(), Acc) -> {ok|stop, Acc}), Acc) -> {ok, Acc}. +enum_docs(_Db, _StartId, _Dir, _Fun, _Acc0) -> +%% couch_db:enum_docs(Db, StartId, Dir, Fun, Acc0). + not_implemented. + +-spec enum_docs_reduce_to_count({any(), any()}) -> non_neg_integer(). +enum_docs_reduce_to_count(_Reductions) -> +%% couch_db:enum_docs_reduce_to_count(Reductions). + not_implemented. + +-spec enum_docs_since(#db{}, update_seq(), fwd | rev, fun((#doc_info{}, + Offset::integer(), Acc) -> {ok|stop, Acc}), Acc) -> {ok, Acc}. +enum_docs_since(_Db, _StartKey, _Dir, _Fun, _Acc0) -> +%% couch_db:enum_docs_since(Db, StartKey, Dir, Fun, Acc0). + not_implemented. + +-spec enum_docs_since_reduce_to_count({any(), any()}) -> non_neg_integer(). +enum_docs_since_reduce_to_count(_Reductions) -> +%% couch_db:enum_docs_since_reduce_to_count(Reductions). + not_implemented. + +-spec get_doc_info(#db{}, docid()) -> {ok, #doc_info{}}. +get_doc_info(_Db, _DocId) -> +%% couch_db:get_doc_info(Db, DocId). + not_implemented. + +-spec get_missing_revs(#db{}, [{docid(), [revision()]}]) -> + {ok, [{docid(), [revision()]}]}. +get_missing_revs(Db, IdsRevs) -> + showroom_doc:get_missing_revs(Db, IdsRevs). + +open_doc(Db, DocId) -> + open_doc(Db, DocId, nil, []). + +-spec open_doc(#db{}, docid(), [any()]) -> {ok, #doc{}} | {not_found, deleted | + missing}. +open_doc(Db, DocId, Options) -> + open_doc(Db, DocId, nil, Options). + +open_doc(Db, DocId, Revs, Options) -> + showroom_doc:open_doc(Db, DocId, Revs, Options). + +-spec open_doc_revs(#db{}, docid(), [revision()], [any()]) -> {ok, [{ok, #doc{}} + | {{not_found, deleted | missing}, revision()}]}. +open_doc_revs(Db, DocId, Revs, Options) -> + open_doc(Db, DocId, Revs, Options). + +-spec purge_docs(#db{}, [{docid(), [revision()]}]) -> + {ok, update_seq(), [{docid(),[revision()]} | {error,purge_during_compaction}]}. +purge_docs(_Db, _IdsRevs) -> +%% couch_db:purge_docs(Db, IdsRevs). + not_implemented. + +update_doc(Db, Doc) -> + update_doc(Db, Doc, []). + +-spec update_doc(#db{}, #doc{}, [any()]) -> {ok, revision()}. +update_doc(Db, Doc, Options) -> + showroom_doc:update_doc(Db, Doc, Options). + +update_docs(Db, Docs, Options) -> + update_docs(Db, Docs, Options, interactive_edit). + +-spec update_docs(#db{}, [#doc{}], [any()], interactive_edit | + replicated_changes) -> {ok, [{ok, revision()}]}. +update_docs(Db, Docs, Options, Type) -> + showroom_doc:update_docs(Db, Docs, Options, Type). + + +%% View + +-spec all_docs_view(response(), #db{}, nil | list(), #view_query_args{}) -> + {ok, any()}. +all_docs_view(Resp, Db, Keys, QueryArgs) -> + showroom_view:all_docs(Resp, Db, Keys, QueryArgs). + +-spec compact_view_group(#db{}, bstring()) -> ok. +compact_view_group(_Db, _DesignId) -> +%% couch_view_compactor:start_compact(Db, DesignId). + not_implemented. + +-spec cleanup_view_index_files(#db{}) -> any(). +cleanup_view_index_files(_Db) -> +%% couch_view:cleanup_index_files(Db). + not_implemented. + +-spec design_view(response(), #db{}, bstring(), bstring(), nil | list(), + #view_query_args{}) -> any(). +design_view(Resp, Db, Id, Name, Keys, QueryArgs) -> + showroom_view:design(Resp, Db, Id, Name, Keys, QueryArgs). + +list_view(Req, Db, DesignId, ViewName, Keys, QueryArgs, QueryServer) -> + showroom_view:list(Req, Db, DesignId, ViewName, Keys, QueryArgs, QueryServer). + +-spec extract_map_view({reduce, any(), bstring(), #view{}}) -> {ok, #view{}}. +extract_map_view(_ReduceView) -> +%% couch_view:extract_map_view(ReduceView). + not_implemented. + +-spec get_map_view(#db{}, bstring(), bstring(), true | false) -> any(). +get_map_view(_Db, _DesignId, _ViewName, _Stale) -> +%% couch_view:get_map_view(Db, DesignId, ViewName, Stale). + not_implemented. + +-spec get_reduce_view(#db{}, bstring(), bstring(), true | false) -> any(). +get_reduce_view(_Db, _DesignId, _ViewName, _Stale) -> +%% couch_view:get_reduce_view(Db, DesignId, ViewName, Stale). + not_implemented. + +-spec get_row_count(#view{}) -> {ok, non_neg_integer()}. +get_row_count(_View) -> +%% couch_view:get_row_count(View). + not_implemented. + +-spec get_temp_map_view(#db{}, bstring(), [any()], bstring()) -> {ok, #view{}, + #group{}}. +get_temp_map_view(_Db, _Language, _DesignOptions, _MapSrc) -> +%% couch_view:get_temp_map_view(Db, Language, DesignOptions, MapSrc). + not_implemented. + +-spec get_temp_reduce_view(#db{}, bstring(), [any()], bstring(), bstring()) -> + {ok, {temp_reduce, #view{}}, #group{}}. +get_temp_reduce_view(_Db, _Language, _DesignOptions, _MapSrc, _RedSrc) -> +%% couch_view:get_temp_reduce_view(Db, Language, DesignOptions, MapSrc, RedSrc). + not_implemented. + +-spec get_view_group_info(#db{}, bstring()) -> {ok, [{atom(), any()}]}. +get_view_group_info(Db, DesignId) -> + showroom_view:group_info(Db, DesignId). + +-spec reduce_to_count({any(), any()}) -> non_neg_integer(). +reduce_to_count(_Reductions) -> +%% couch_view:reduce_to_count(Reductions). + not_implemented. + +-spec view_fold(#view{}, any(), fwd | rev, fun(({{Key::any(), docid()}, + Value::any()}, OffsetReds::any(), Acc) -> {ok|stop, Acc}), Acc) -> {ok, Acc}. +view_fold(_View, _Start, _Dir, _Fun, _Acc0) -> +%% couch_view:fold(View, Start, Dir, Fun, Acc0). + not_implemented. + +-spec view_fold_reduce({reduce | temp_reduce, #view{}}, fwd | rev, any(), any(), + fun(({Key1::any(), any()}, {Key2::any(), any()}) -> boolean()), + fun(({Key::any(), Reduction::any(), Acc}) -> {ok|stop, Acc}), Acc) -> + {ok, Acc}. +view_fold_reduce(_View, _Dir, _Start, _End, _GroupFun, _ResponseFun, _Acc0) -> +%% couch_view:fold_reduce(View, Dir, Start, End, GroupFun, ResponseFun, Acc0). + not_implemented. diff --git a/src/fabric_create.erl b/src/fabric_create.erl new file mode 100644 index 00000000..b47cd768 --- /dev/null +++ b/src/fabric_create.erl @@ -0,0 +1,141 @@ +-module(fabric_create). +-author('Brad Anderson '). + +-include("../../couch/src/couch_db.hrl"). + +%% api +-export([create_db/2]). + +-type part() :: integer(). +-type ref_node_part() :: {reference(), node(), part()}. +-type tref() :: reference(). +-type np() :: {node(), part()}. +-type np_acc() :: [{np(), any()}]. + + +%% ===================== +%% api +%% ===================== + +%% @doc Create a new database, and all its partition files across the cluster +%% Options is proplist with user_ctx, n, q +-spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. +create_db(DbName, Options) -> + RefNodePart = send_create_calls(DbName, Options), + {ok, Results} = create_db_loop(RefNodePart), + case create_results(Results, RefNodePart) of + ok -> {ok, #db{name=DbName}}; + Other -> {error, Other} + end. + + +%% ===================== +%% internal +%% ===================== + +%% @doc create the partitions on all appropriate nodes (rexi calls) +-spec send_create_calls(binary(), list()) -> [{reference(), np()}]. +send_create_calls(DbName, Options) -> + Fullmap = partitions:fullmap(DbName, Options), + lists:map(fun({Node, Part}) -> + ShardName = showroom_utils:shard_name(Part, DbName), + Ref = rexi:async_server_call({couch_server, Node}, + {create, ShardName, Options}), + {Ref, {Node, Part}} + end, Fullmap). + +%% @doc set up the receive loop with an overall timeout +-spec create_db_loop([ref_node_part()]) -> {ok, np_acc()}. +create_db_loop(RefNodePart) -> + TimeoutRef = erlang:make_ref(), + {ok, TRef} = timer:send_after(5000, {timeout, TimeoutRef}), + Results = create_db_loop(RefNodePart, TimeoutRef, []), + timer:cancel(TRef), + Results. + +%% @doc create_db receive loop +-spec create_db_loop([ref_node_part()], tref(), np_acc()) -> + np_acc() | {ok, np_acc()}. +create_db_loop(_,_,{ok, Acc}) -> {ok, Acc}; +create_db_loop(RefNodePart, TimeoutRef, AccIn) -> + receive + {Ref, {ok, MainPid}} when is_reference(Ref) -> + % for dev only, close the Fd + gen_server:call({couch_server, node(MainPid)}, {force_close, MainPid}), + + AccOut = check_all_parts(Ref, RefNodePart, AccIn, ok), + create_db_loop(RefNodePart, TimeoutRef, AccOut); + {Ref, Reply} when is_reference(Ref) -> + AccOut = check_all_parts(Ref, RefNodePart, AccIn, Reply), + create_db_loop(RefNodePart, TimeoutRef, AccOut); + {timeout, TimeoutRef} -> + {error, timeout} + end. + +-spec create_results(np_acc(), [ref_node_part()]) -> ok | create_quorum_error. +create_results(Results, RefNodePart) -> + NPs = create_result(Results, []), + DistinctNPs = distinct_parts(RefNodePart), + if + NPs =:= DistinctNPs -> ok; + true -> create_quorum_error + end. + +-spec create_result(np_acc(), [np()]) -> [np()] | file_exists. +create_result([], Acc) -> + Acc; +create_result([{NP, ok}|Rest], Acc) -> + create_result(Rest, [NP|Acc]); +create_result([{_NP, {error, file_exists}}|_Rest], _Acc) -> + {error, file_exists}; % if any replies were file_exists, return that +create_result([{{_N,_P}, Result}|Rest], Acc) -> + showroom_log:message(error, "create_db error: ~p", [Result]), + create_result(Rest, Acc). + +check_all_parts(Ref, RefNodePart, Acc, Reply) -> + case couch_util:get_value(Ref, RefNodePart) of + {Node, Part} -> + case lists:keyfind(1, {Node, Part}, Acc) of + true -> Acc; % already present... that's odd + _ -> + NewAcc = [{{Node, Part}, Reply} | Acc], + case length(NewAcc) >= length(RefNodePart) of + true -> {ok, NewAcc}; + _ -> NewAcc + end + end; + _ -> Acc % ignore a non-matching Ref + end. + +%% @doc check that we have a good reply from each partition. +%% If we do, return {ok, Acc}, if we don't, return Acc of partitions +%% Three 'case' statements and one 'if', a personal best. fml +%% @end +% check_distinct_parts(Ref, RefNodePart, Acc, Msg) -> +% Parts = distinct_parts(RefNodePart), +% case couch_util:get_value(Ref, RefNodePart) of +% {Node, Part} -> +% case lists:member(Part, Acc) of +% true -> Acc; +% _ -> +% case Msg of +% ok -> +% NewAcc = lists:usort([Part|Acc]), +% if +% Parts =:= NewAcc -> {ok, NewAcc}; +% true -> NewAcc +% end; +% _ -> +% Hex = showroom_utils:int_to_hexstr(Part), +% showroom_log:message(error, +% "create_db reply error: ~p from ~p ~p", [Msg, Node, Hex]), +% Acc +% end +% end; +% _ -> Acc % ignore a non-matching Ref +% end. + +distinct_parts(RefNodePart) -> + {_Refs, NPs} = lists:unzip(RefNodePart), + {_Nodes, Parts} = lists:unzip(NPs), + lists:usort(Parts). -- cgit v1.2.3 From 322ef619409da556cd36b16315c332417a4eba1d Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Tue, 25 May 2010 12:46:11 -0400 Subject: fix create_db bug --- src/fabric_create.erl | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/fabric_create.erl b/src/fabric_create.erl index b47cd768..245ae2a2 100644 --- a/src/fabric_create.erl +++ b/src/fabric_create.erl @@ -54,13 +54,15 @@ create_db_loop(RefNodePart) -> Results. %% @doc create_db receive loop +%% Acc is either an accumulation of responses, or if we've received all +%% responses, it's {ok, Responses} -spec create_db_loop([ref_node_part()], tref(), np_acc()) -> np_acc() | {ok, np_acc()}. create_db_loop(_,_,{ok, Acc}) -> {ok, Acc}; create_db_loop(RefNodePart, TimeoutRef, AccIn) -> receive {Ref, {ok, MainPid}} when is_reference(Ref) -> - % for dev only, close the Fd + % for dev only, close the Fd TODO: remove me gen_server:call({couch_server, node(MainPid)}, {force_close, MainPid}), AccOut = check_all_parts(Ref, RefNodePart, AccIn, ok), @@ -72,20 +74,25 @@ create_db_loop(RefNodePart, TimeoutRef, AccIn) -> {error, timeout} end. +%% @doc check the results of the create replies +%% If we have a good reply from each partition, return ok -spec create_results(np_acc(), [ref_node_part()]) -> ok | create_quorum_error. create_results(Results, RefNodePart) -> - NPs = create_result(Results, []), - DistinctNPs = distinct_parts(RefNodePart), + ResultParts = create_result(Results, []), + DistinctParts = distinct_parts(RefNodePart), if - NPs =:= DistinctNPs -> ok; - true -> create_quorum_error + ResultParts =:= DistinctParts -> ok; + true -> + ?debugFmt("~nResultParts: ~p~nDistinctParts: ~p~n", + [ResultParts, DistinctParts]), + create_quorum_error end. -spec create_result(np_acc(), [np()]) -> [np()] | file_exists. create_result([], Acc) -> - Acc; -create_result([{NP, ok}|Rest], Acc) -> - create_result(Rest, [NP|Acc]); + lists:usort(Acc); +create_result([{{_N,P}, ok}|Rest], Acc) -> + create_result(Rest, [P|Acc]); create_result([{_NP, {error, file_exists}}|_Rest], _Acc) -> {error, file_exists}; % if any replies were file_exists, return that create_result([{{_N,_P}, Result}|Rest], Acc) -> @@ -95,7 +102,7 @@ create_result([{{_N,_P}, Result}|Rest], Acc) -> check_all_parts(Ref, RefNodePart, Acc, Reply) -> case couch_util:get_value(Ref, RefNodePart) of {Node, Part} -> - case lists:keyfind(1, {Node, Part}, Acc) of + case lists:keyfind({Node, Part}, 1, Acc) of true -> Acc; % already present... that's odd _ -> NewAcc = [{{Node, Part}, Reply} | Acc], -- cgit v1.2.3 From 4e75ad7622cb190616c653dde5d7a5625245fce0 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Tue, 25 May 2010 15:21:45 -0400 Subject: all_databases now working with/without Customer param, types moved to mem.hrl, view updater thwarted for updates to dbs db. --- src/fabric.erl | 6 +++++- src/fabric_api.erl | 2 +- src/fabric_create.erl | 20 +++++++++----------- src/fabric_info.erl | 23 +++++++++++++++++++++++ 4 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 src/fabric_info.erl diff --git a/src/fabric.erl b/src/fabric.erl index d588265f..3e5d9dd8 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,6 +1,10 @@ -module(fabric). --export([create_db/2]). +-export([all_databases/1, create_db/2]). + + +all_databases(Customer) -> + fabric_info:all_databases(Customer). create_db(DbName, Options) -> fabric_create:create_db(DbName, Options). diff --git a/src/fabric_api.erl b/src/fabric_api.erl index a26aa7de..f932ff25 100644 --- a/src/fabric_api.erl +++ b/src/fabric_api.erl @@ -23,7 +23,7 @@ db_path(RawUri, Customer) -> -spec all_databases(string()) -> {ok, [bstring()]}. all_databases(Customer) -> - showroom_db:all_databases(Customer). + fabric:all_databases(Customer). -spec create_db(bstring(), [any()]) -> {ok, #db{}} | {error, any()}. create_db(DbName, Options) -> diff --git a/src/fabric_create.erl b/src/fabric_create.erl index 245ae2a2..986e4c2a 100644 --- a/src/fabric_create.erl +++ b/src/fabric_create.erl @@ -2,16 +2,11 @@ -author('Brad Anderson '). -include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). %% api -export([create_db/2]). --type part() :: integer(). --type ref_node_part() :: {reference(), node(), part()}. --type tref() :: reference(). --type np() :: {node(), part()}. --type np_acc() :: [{np(), any()}]. - %% ===================== %% api @@ -21,10 +16,14 @@ %% Options is proplist with user_ctx, n, q -spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. create_db(DbName, Options) -> - RefNodePart = send_create_calls(DbName, Options), + Fullmap = partitions:fullmap(DbName, Options), + {ok, FullNodes} = mem3:fullnodes(), + RefNodePart = send_create_calls(DbName, Options, Fullmap), {ok, Results} = create_db_loop(RefNodePart), case create_results(Results, RefNodePart) of - ok -> {ok, #db{name=DbName}}; + ok -> + partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), + {ok, #db{name=DbName}}; Other -> {error, Other} end. @@ -34,9 +33,8 @@ create_db(DbName, Options) -> %% ===================== %% @doc create the partitions on all appropriate nodes (rexi calls) --spec send_create_calls(binary(), list()) -> [{reference(), np()}]. -send_create_calls(DbName, Options) -> - Fullmap = partitions:fullmap(DbName, Options), +-spec send_create_calls(binary(), list(), [mem_node()]) -> [{reference(), np()}]. +send_create_calls(DbName, Options, Fullmap) -> lists:map(fun({Node, Part}) -> ShardName = showroom_utils:shard_name(Part, DbName), Ref = rexi:async_server_call({couch_server, Node}, diff --git a/src/fabric_info.erl b/src/fabric_info.erl new file mode 100644 index 00000000..95408b6f --- /dev/null +++ b/src/fabric_info.erl @@ -0,0 +1,23 @@ +-module(fabric_info). + +-export([all_databases/1]). + +-include("../../couch/src/couch_db.hrl"). + +%% @doc gets all databases in the cluster. +-spec all_databases(binary() | []) -> [binary()]. +all_databases([]) -> + Dbs = ets:foldl(fun({DbName, _}, Acc0) -> + [DbName | Acc0] + end, [], dbs_cache), + {ok, Dbs}; +all_databases(Customer) -> + ?debugFmt("~nCustomer: ~p~n", [Customer]), + Dbs = ets:foldl(fun({DbName, _}, Acc0) -> + DbNameStr = ?b2l(DbName), + case string:str(DbNameStr, Customer) of + 1 -> [DbNameStr | Acc0]; + _ -> Acc0 + end + end, [], dbs_cache), + {ok, Dbs}. -- cgit v1.2.3 From ddbf9b2a6764389b539b7920a0da3abe08a64e77 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Wed, 26 May 2010 10:51:10 -0400 Subject: add delete_db, open_db, and open_doc calls to fabric front-end --- src/fabric.erl | 13 +++- src/fabric_api.erl | 172 +++++------------------------------------------------ 2 files changed, 26 insertions(+), 159 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index 3e5d9dd8..5aefed5b 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,10 +1,21 @@ -module(fabric). --export([all_databases/1, create_db/2]). +-export([all_databases/1, create_db/2, delete_db/2, open_db/2, open_doc/4]). +%% maybe this goes away, and these are called directly in their own modules in +%% fabric_api ?? all_databases(Customer) -> fabric_info:all_databases(Customer). create_db(DbName, Options) -> fabric_create:create_db(DbName, Options). + +delete_db(DbName, Options) -> + fabric_delete:delete_db(DbName, Options). + +open_db(DbName, Options) -> + fabric_open:open_db(DbName, Options). + +open_doc(Db, DocId, Revs, Options) -> + fabric_open:open_doc(Db, DocId, Revs, Options). diff --git a/src/fabric_api.erl b/src/fabric_api.erl index f932ff25..e399e048 100644 --- a/src/fabric_api.erl +++ b/src/fabric_api.erl @@ -31,11 +31,11 @@ create_db(DbName, Options) -> -spec delete_db(bstring(), [any()]) -> ok | not_found | {error, atom()}. delete_db(DbName, Options) -> - showroom_db:delete_db(DbName, Options). + fabric:delete_db(DbName, Options). -spec open_db(bstring(), [any()]) -> {ok, #db{}} | {error, any()}. open_db(DbName, Options) -> - showroom_db:open_db(DbName, Options). + fabric:open_db(DbName, Options). -spec close_db(#db{}) -> ok. close_db(Db) -> @@ -45,26 +45,6 @@ close_db(Db) -> get_db_info(Db, Customer) -> showroom_db:get_db_info(Db, Customer). --spec get_committed_update_seq(#db{}) -> update_seq(). -get_committed_update_seq(_Db) -> -%% couch_db:get_committed_update_seq(Db). - not_implemented. - --spec get_purge_seq(#db{}) -> update_seq(). -get_purge_seq(_Db) -> -%% couch_db:get_purge_seq(Db). - not_implemented. - --spec get_update_seq(#db{}) -> update_seq(). -get_update_seq(_Db) -> -%% couch_db:get_update_seq(Db). - not_implemented. - --spec compact_db(#db{}) -> ok. -compact_db(_Db) -> -%% couch_db:start_compact(Db). - not_implemented. - -spec replicate_db(ejson(), #user_ctx{}) -> {ok, ejson()}. replicate_db(PostBody, UserCtx) -> showroom_rep:replicate(PostBody, UserCtx). @@ -74,96 +54,33 @@ ensure_full_commit(Db) -> showroom_db:ensure_full_commit(Db), {ok, <<"0">>}. --spec increment_update_seq(#db{}) -> {ok, update_seq()}. -increment_update_seq(_Db) -> -%% couch_db:increment_update_seq(Db). - not_implemented. - --spec get_admins(#db{}) -> [any()]. -get_admins(_Db) -> -%% couch_db:get_admins(Db). - not_implemented. - --spec set_admins(#db{}, [any()]) -> ok. -set_admins(_Db, _Admins) -> -%% couch_db:set_admins(Db, Admins). - not_implemented. - --spec get_revs_limit(#db{}) -> pos_integer(). -get_revs_limit(_Db) -> -%% couch_db:get_revs_limit(Db). - not_implemented. - --spec set_revs_limit(#db{}, pos_integer()) -> ok. -set_revs_limit(_Db, _Limit) -> -%% couch_db:set_revs_limit(Db, Limit). - not_implemented. - %% Document att_receiver(Req, Length) -> showroom_att:receiver(Req, Length). --spec changes_since(#db{}, main_only | all_docs, update_seq(), - fun(([#doc_info{}], Acc) -> {ok, Acc}), Acc) -> {ok, Acc}. -changes_since(_Db, _Style, _StartSeq, _Fun, _Acc0) -> -%% couch_db:changes_since(Db, Style, StartSeq, Fun, Acc0). - not_implemented. - --spec enum_docs(#db{}, docid(), fwd | rev, fun((#full_doc_info{}, - Offset::integer(), Acc) -> {ok|stop, Acc}), Acc) -> {ok, Acc}. -enum_docs(_Db, _StartId, _Dir, _Fun, _Acc0) -> -%% couch_db:enum_docs(Db, StartId, Dir, Fun, Acc0). - not_implemented. - --spec enum_docs_reduce_to_count({any(), any()}) -> non_neg_integer(). -enum_docs_reduce_to_count(_Reductions) -> -%% couch_db:enum_docs_reduce_to_count(Reductions). - not_implemented. - --spec enum_docs_since(#db{}, update_seq(), fwd | rev, fun((#doc_info{}, - Offset::integer(), Acc) -> {ok|stop, Acc}), Acc) -> {ok, Acc}. -enum_docs_since(_Db, _StartKey, _Dir, _Fun, _Acc0) -> -%% couch_db:enum_docs_since(Db, StartKey, Dir, Fun, Acc0). - not_implemented. - --spec enum_docs_since_reduce_to_count({any(), any()}) -> non_neg_integer(). -enum_docs_since_reduce_to_count(_Reductions) -> -%% couch_db:enum_docs_since_reduce_to_count(Reductions). - not_implemented. - --spec get_doc_info(#db{}, docid()) -> {ok, #doc_info{}}. -get_doc_info(_Db, _DocId) -> -%% couch_db:get_doc_info(Db, DocId). - not_implemented. - -spec get_missing_revs(#db{}, [{docid(), [revision()]}]) -> {ok, [{docid(), [revision()]}]}. get_missing_revs(Db, IdsRevs) -> showroom_doc:get_missing_revs(Db, IdsRevs). -open_doc(Db, DocId) -> - open_doc(Db, DocId, nil, []). +open_doc(DbName, DocId) -> + open_doc(DbName, DocId, nil, []). --spec open_doc(#db{}, docid(), [any()]) -> {ok, #doc{}} | {not_found, deleted | - missing}. -open_doc(Db, DocId, Options) -> - open_doc(Db, DocId, nil, Options). +-spec open_doc(bstring(), docid(), [any()]) -> + {ok, #doc{}} | {not_found, deleted | missing}. +open_doc(DbName, DocId, Options) -> + open_doc(DbName, DocId, nil, Options). -open_doc(Db, DocId, Revs, Options) -> - showroom_doc:open_doc(Db, DocId, Revs, Options). +open_doc(DbName, DocId, Revs, Options) -> + fabric:open_doc(DbName, DocId, Revs, Options). --spec open_doc_revs(#db{}, docid(), [revision()], [any()]) -> {ok, [{ok, #doc{}} +-spec open_doc_revs(bstring(), docid(), [revision()], [any()]) -> + {ok, [{ok, #doc{}} | {{not_found, deleted | missing}, revision()}]}. -open_doc_revs(Db, DocId, Revs, Options) -> - open_doc(Db, DocId, Revs, Options). - --spec purge_docs(#db{}, [{docid(), [revision()]}]) -> - {ok, update_seq(), [{docid(),[revision()]} | {error,purge_during_compaction}]}. -purge_docs(_Db, _IdsRevs) -> -%% couch_db:purge_docs(Db, IdsRevs). - not_implemented. +open_doc_revs(DbName, DocId, Revs, Options) -> + open_doc(DbName, DocId, Revs, Options). update_doc(Db, Doc) -> update_doc(Db, Doc, []). @@ -188,16 +105,6 @@ update_docs(Db, Docs, Options, Type) -> all_docs_view(Resp, Db, Keys, QueryArgs) -> showroom_view:all_docs(Resp, Db, Keys, QueryArgs). --spec compact_view_group(#db{}, bstring()) -> ok. -compact_view_group(_Db, _DesignId) -> -%% couch_view_compactor:start_compact(Db, DesignId). - not_implemented. - --spec cleanup_view_index_files(#db{}) -> any(). -cleanup_view_index_files(_Db) -> -%% couch_view:cleanup_index_files(Db). - not_implemented. - -spec design_view(response(), #db{}, bstring(), bstring(), nil | list(), #view_query_args{}) -> any(). design_view(Resp, Db, Id, Name, Keys, QueryArgs) -> @@ -206,57 +113,6 @@ design_view(Resp, Db, Id, Name, Keys, QueryArgs) -> list_view(Req, Db, DesignId, ViewName, Keys, QueryArgs, QueryServer) -> showroom_view:list(Req, Db, DesignId, ViewName, Keys, QueryArgs, QueryServer). --spec extract_map_view({reduce, any(), bstring(), #view{}}) -> {ok, #view{}}. -extract_map_view(_ReduceView) -> -%% couch_view:extract_map_view(ReduceView). - not_implemented. - --spec get_map_view(#db{}, bstring(), bstring(), true | false) -> any(). -get_map_view(_Db, _DesignId, _ViewName, _Stale) -> -%% couch_view:get_map_view(Db, DesignId, ViewName, Stale). - not_implemented. - --spec get_reduce_view(#db{}, bstring(), bstring(), true | false) -> any(). -get_reduce_view(_Db, _DesignId, _ViewName, _Stale) -> -%% couch_view:get_reduce_view(Db, DesignId, ViewName, Stale). - not_implemented. - --spec get_row_count(#view{}) -> {ok, non_neg_integer()}. -get_row_count(_View) -> -%% couch_view:get_row_count(View). - not_implemented. - --spec get_temp_map_view(#db{}, bstring(), [any()], bstring()) -> {ok, #view{}, - #group{}}. -get_temp_map_view(_Db, _Language, _DesignOptions, _MapSrc) -> -%% couch_view:get_temp_map_view(Db, Language, DesignOptions, MapSrc). - not_implemented. - --spec get_temp_reduce_view(#db{}, bstring(), [any()], bstring(), bstring()) -> - {ok, {temp_reduce, #view{}}, #group{}}. -get_temp_reduce_view(_Db, _Language, _DesignOptions, _MapSrc, _RedSrc) -> -%% couch_view:get_temp_reduce_view(Db, Language, DesignOptions, MapSrc, RedSrc). - not_implemented. - -spec get_view_group_info(#db{}, bstring()) -> {ok, [{atom(), any()}]}. get_view_group_info(Db, DesignId) -> showroom_view:group_info(Db, DesignId). - --spec reduce_to_count({any(), any()}) -> non_neg_integer(). -reduce_to_count(_Reductions) -> -%% couch_view:reduce_to_count(Reductions). - not_implemented. - --spec view_fold(#view{}, any(), fwd | rev, fun(({{Key::any(), docid()}, - Value::any()}, OffsetReds::any(), Acc) -> {ok|stop, Acc}), Acc) -> {ok, Acc}. -view_fold(_View, _Start, _Dir, _Fun, _Acc0) -> -%% couch_view:fold(View, Start, Dir, Fun, Acc0). - not_implemented. - --spec view_fold_reduce({reduce | temp_reduce, #view{}}, fwd | rev, any(), any(), - fun(({Key1::any(), any()}, {Key2::any(), any()}) -> boolean()), - fun(({Key::any(), Reduction::any(), Acc}) -> {ok|stop, Acc}), Acc) -> - {ok, Acc}. -view_fold_reduce(_View, _Dir, _Start, _End, _GroupFun, _ResponseFun, _Acc0) -> -%% couch_view:fold_reduce(View, Dir, Start, End, GroupFun, ResponseFun, Acc0). - not_implemented. -- cgit v1.2.3 From 64483a26df653edb9daa490a6c2b357a35f9f804 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Wed, 26 May 2010 10:52:54 -0400 Subject: adding delete and open modules to fabric --- ebin/fabric.app | 4 +- src/fabric_delete.erl | 148 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/fabric_open.erl | 61 +++++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 src/fabric_delete.erl create mode 100644 src/fabric_open.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index 76de79ff..76dff491 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -6,7 +6,9 @@ {modules, [ fabric, fabric_api, - fabric_create + fabric_create, + fabric_delete, + fabric_info ]}, {registered, []}, {included_applications, []}, diff --git a/src/fabric_delete.erl b/src/fabric_delete.erl new file mode 100644 index 00000000..e77f813f --- /dev/null +++ b/src/fabric_delete.erl @@ -0,0 +1,148 @@ +-module(fabric_delete). +-author('Brad Anderson '). + +-include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). + +%% api +-export([delete_db/2]). + + +%% ===================== +%% api +%% ===================== + +%% @doc Delete a new database, and all its partition files across the cluster +%% Options is proplist with user_ctx, n, q +-spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. +delete_db(DbName, Options) -> + Fullmap = partitions:fullmap(DbName, Options), + RefNodePart = send_delete_calls(DbName, Options, Fullmap), + {ok, Results} = delete_db_loop(RefNodePart), + delete_results(Results, RefNodePart). + + +%delete_db(DbName, Options) -> +% ResolveFun = fun(_Good) -> true end, +% case cluster_ops:all_parts({dynomite_couch_api,delete_db,[DbName, Options]}, +% d, true, ResolveFun) of +% {ok, true} -> ok; +% [{error, d_quorum_not_met}, {good, _Good}, {bad, Bad}] -> +% showroom_utils:first_bad(Bad); +% [{error, Error}, {good, _Good}, {bad, Bad}] -> +% {Error, showroom_utils:first_bad(Bad)}; +% Other -> +% ?debugFmt("~nOther: ~p~n", [Other]), +% Other +% end. + +%% ===================== +%% internal +%% ===================== + +%% @doc delete the partitions on all appropriate nodes (rexi calls) +-spec send_delete_calls(binary(), list(), [mem_node()]) -> [{reference(), np()}]. +send_delete_calls(DbName, Options, Fullmap) -> + lists:map(fun({Node, Part}) -> + ShardName = showroom_utils:shard_name(Part, DbName), + Ref = rexi:async_server_call({couch_server, Node}, + {delete, ShardName, Options}), + {Ref, {Node, Part}} + end, Fullmap). + +%% @doc set up the receive loop with an overall timeout +-spec delete_db_loop([ref_node_part()]) -> {ok, np_acc()}. +delete_db_loop(RefNodePart) -> + TimeoutRef = erlang:make_ref(), + {ok, TRef} = timer:send_after(5000, {timeout, TimeoutRef}), + Results = delete_db_loop(RefNodePart, TimeoutRef, []), + timer:cancel(TRef), + Results. + +%% @doc delete_db receive loop +%% Acc is either an accumulation of responses, or if we've received all +%% responses, it's {ok, Responses} +-spec delete_db_loop([ref_node_part()], tref(), np_acc()) -> + np_acc() | {ok, np_acc()}. +delete_db_loop(_,_,{ok, Acc}) -> {ok, Acc}; +delete_db_loop(RefNodePart, TimeoutRef, AccIn) -> + receive + {Ref, {ok, deleted}} when is_reference(Ref) -> + AccOut = check_all_parts(Ref, RefNodePart, AccIn, ok), + delete_db_loop(RefNodePart, TimeoutRef, AccOut); + {Ref, Reply} when is_reference(Ref) -> + AccOut = check_all_parts(Ref, RefNodePart, AccIn, Reply), + delete_db_loop(RefNodePart, TimeoutRef, AccOut); + {timeout, TimeoutRef} -> + {error, timeout} + end. + +%% @doc check the results of the delete replies +%% If we have a good reply from all partitions, return ok +-spec delete_results(np_acc(), [ref_node_part()]) -> + ok | {error, delete_quorum_error}. +delete_results(Results, RefNodePart) -> + ResultNPs = delete_result(Results, []), + AllNPs = all_nodes_parts(RefNodePart), + if + ResultNPs =:= AllNPs -> ok; + true -> {error, delete_quorum_error} + end. + +-spec delete_result(np_acc(), [np()]) -> [np()] | file_exists. +delete_result([], Acc) -> + lists:sort(Acc); +delete_result([{NP, ok}|Rest], Acc) -> + delete_result(Rest, [NP|Acc]); +delete_result([{_NP, {error, file_exists}}|_Rest], _Acc) -> + {error, file_exists}; % if any replies were file_exists, return that +delete_result([{{_N,_P}, Result}|Rest], Acc) -> + ?LOG_ERROR("delete_db error: ~p", [Result]), + delete_result(Rest, Acc). + +check_all_parts(Ref, RefNodePart, Acc, Reply) -> + case couch_util:get_value(Ref, RefNodePart) of + {Node, Part} -> + case lists:keyfind({Node, Part}, 1, Acc) of + true -> Acc; % already present... that's odd + _ -> + NewAcc = [{{Node, Part}, Reply} | Acc], + case length(NewAcc) >= length(RefNodePart) of + true -> {ok, NewAcc}; + _ -> NewAcc + end + end; + _ -> Acc % ignore a non-matching Ref + end. + +%% @doc check that we have a good reply from each partition. +%% If we do, return {ok, Acc}, if we don't, return Acc of partitions +%% Three 'case' statements and one 'if', a personal best. fml +%% @end +% check_distinct_parts(Ref, RefNodePart, Acc, Msg) -> +% Parts = distinct_parts(RefNodePart), +% case couch_util:get_value(Ref, RefNodePart) of +% {Node, Part} -> +% case lists:member(Part, Acc) of +% true -> Acc; +% _ -> +% case Msg of +% ok -> +% NewAcc = lists:usort([Part|Acc]), +% if +% Parts =:= NewAcc -> {ok, NewAcc}; +% true -> NewAcc +% end; +% _ -> +% Hex = showroom_utils:int_to_hexstr(Part), +% showroom_log:message(error, +% "delete_db reply error: ~p from ~p ~p", [Msg, Node, Hex]), +% Acc +% end +% end; +% _ -> Acc % ignore a non-matching Ref +% end. + +all_nodes_parts(RefNodePart) -> + {_Refs, NPs} = lists:unzip(RefNodePart), + lists:sort(NPs). diff --git a/src/fabric_open.erl b/src/fabric_open.erl new file mode 100644 index 00000000..a8c0204e --- /dev/null +++ b/src/fabric_open.erl @@ -0,0 +1,61 @@ +-module(fabric_open). + +-export([open_doc/4]). +-export([open_doc_endpoint/4]). + +-include("../../couch/src/couch_db.hrl"). + + +% open_db(<<"S", ShardFileName/binary>> = Name, Options) -> +% case couch_db:open(ShardFileName, Options) of +% {ok, Db} -> +% {ok, Db#db{name = Name}}; +% {not_found, no_db_file} -> +% {not_found, no_db_file} +% end; +% +% open_db(DbName, Options) -> +% Part = case lists:keyfind(node(), 1, membership2:all_nodes_parts(false)) of +% {_, P} -> P; +% _ -> throw({node_has_no_parts, node()}) +% end, +% ShardName = partitions:shard_name(Part, DbName), +% open_db(<<"S", ShardName/binary>>, Options). + + +open_doc(DbName, DocId, Revs, Options) -> + NPs = partitions:key_nodes_parts(DbName, DocId), + ?debugFmt("~nNPs: ~p~n", [NPs]), + ok. + + +open_doc_endpoint(DbName, DocId, Revs, Options) -> + case couch_db:open(DbName, []) of + {ok, Db} -> + try + couch_api:open_doc(Db, DocId, Revs, Options) + after + couch_db:close(Db) + end; + {not_found, no_db_file} -> + throw({not_found, <<"The database does not exist.">>}); + Error -> + throw(Error) + end. + + + +% open_doc(Db, DocId, Revs, Options) -> +% {R,N} = get_quorum_constants(r, Options), +% case cluster_ops:key_lookup(DocId, {dynomite_couch_api, get, +% [Db, DocId, Revs, Options]}, r, R, N) of +% {ok, [Doc|_Rest]} -> +% {ok, Doc}; +% {ok, {not_found, Reason}, _Reasons} -> +% {not_found, Reason}; +% [{error, Error}, {good, Good}, {bad, Bad}] -> +% showroom_quorum_utils:handle_error(Error, Good, Bad); +% Other -> +% ?LOG_DEBUG("~nopen_doc Other: ~p~n", [Other]), +% throw(Other) +% end. -- cgit v1.2.3 From 0b8e2d0bb9bed5bac6302c8f22737c37a02bcc68 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Wed, 26 May 2010 15:12:01 -0400 Subject: adding fabric rpc module --- src/fabric_rpc.erl | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 src/fabric_rpc.erl diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl new file mode 100644 index 00000000..fd54827c --- /dev/null +++ b/src/fabric_rpc.erl @@ -0,0 +1,49 @@ +-module(fabric_rpc). + +-export([open_doc/4, get_doc_info/1]). +-export([receive_loop/3]). + +-include("../../dynomite/include/membership.hrl"). + + +open_doc(DbName, DocId, Revs, Options) -> + case couch_db:open(DbName, []) of + {ok, Db} -> + try + couch_api:open_doc(Db, DocId, Revs, Options) + after + couch_db:close(Db) + end; + {not_found, no_db_file} -> + throw({not_found, <<"The database does not exist.">>}); + Error -> + throw(Error) + end. + +get_doc_info(DbName) -> + case couch_db:open(DbName, []) of + {ok, Db} -> + try + couch_db:get_db_info(Db) + after + couch_db:close(Db) + end; + {not_found, no_db_file} -> + throw({not_found, <<"The database does not exist.">>}); + Error -> + throw(Error) + end. + +%% +%% helper funs +%% + +%% @doc set up the receive loop with an overall timeout +-spec receive_loop([ref_part_map()], integer(), function()) -> {ok, beg_acc()}. +receive_loop(RefPartMap, Timeout, Loop) -> + TimeoutRef = erlang:make_ref(), + {ok, TRef} = timer:send_after(Timeout, {timeout, TimeoutRef}), + Results = Loop(RefPartMap, TimeoutRef, []), + timer:cancel(TRef), + Results. +%{Status, Info} = couch_db:get_db_info(Shard), -- cgit v1.2.3 From 7801b1fa879a752d11d538f35ee4905609779801 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Wed, 26 May 2010 15:21:51 -0400 Subject: rest of recent fabric changes. switch to #part{} record, add get_db_info work (not functional), and change others around to use new fullmap --- ebin/fabric.app | 3 +- src/fabric.erl | 6 ++- src/fabric_api.erl | 2 +- src/fabric_create.erl | 114 +++++++++++++++++--------------------------------- src/fabric_info.erl | 104 +++++++++++++++++++++++++++++++++++++++++---- src/fabric_open.erl | 21 ++-------- 6 files changed, 148 insertions(+), 102 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index 76dff491..73943af5 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -8,7 +8,8 @@ fabric_api, fabric_create, fabric_delete, - fabric_info + fabric_info, + fabric_open ]}, {registered, []}, {included_applications, []}, diff --git a/src/fabric.erl b/src/fabric.erl index 5aefed5b..a54264dc 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,6 +1,7 @@ -module(fabric). --export([all_databases/1, create_db/2, delete_db/2, open_db/2, open_doc/4]). +-export([all_databases/1, create_db/2, delete_db/2, open_db/2, open_doc/4, + get_db_info/2]). %% maybe this goes away, and these are called directly in their own modules in %% fabric_api ?? @@ -19,3 +20,6 @@ open_db(DbName, Options) -> open_doc(Db, DocId, Revs, Options) -> fabric_open:open_doc(Db, DocId, Revs, Options). + +get_db_info(DbName, Customer) -> + fabric_info:get_db_info(DbName, Customer). diff --git a/src/fabric_api.erl b/src/fabric_api.erl index e399e048..53dc96e3 100644 --- a/src/fabric_api.erl +++ b/src/fabric_api.erl @@ -43,7 +43,7 @@ close_db(Db) -> -spec get_db_info(#db{}, bstring()) -> {ok, [{atom(), any()}]}. get_db_info(Db, Customer) -> - showroom_db:get_db_info(Db, Customer). + fabric:get_db_info(Db, Customer). -spec replicate_db(ejson(), #user_ctx{}) -> {ok, ejson()}. replicate_db(PostBody, UserCtx) -> diff --git a/src/fabric_create.erl b/src/fabric_create.erl index 986e4c2a..1143d080 100644 --- a/src/fabric_create.erl +++ b/src/fabric_create.erl @@ -18,9 +18,10 @@ create_db(DbName, Options) -> Fullmap = partitions:fullmap(DbName, Options), {ok, FullNodes} = mem3:fullnodes(), - RefNodePart = send_create_calls(DbName, Options, Fullmap), - {ok, Results} = create_db_loop(RefNodePart), - case create_results(Results, RefNodePart) of + RefPartMap = send_create_calls(DbName, Options, Fullmap), + {ok, Results} = fabric_rpc:receive_loop(RefPartMap, 5000, + fun create_db_loop/3), + case create_results(Results, RefPartMap) of ok -> partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), {ok, #db{name=DbName}}; @@ -33,78 +34,69 @@ create_db(DbName, Options) -> %% ===================== %% @doc create the partitions on all appropriate nodes (rexi calls) --spec send_create_calls(binary(), list(), [mem_node()]) -> [{reference(), np()}]. +-spec send_create_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. send_create_calls(DbName, Options, Fullmap) -> - lists:map(fun({Node, Part}) -> - ShardName = showroom_utils:shard_name(Part, DbName), + lists:map(fun(#part{node=Node, b=Beg} = Part) -> + ShardName = showroom_utils:shard_name(Beg, DbName), Ref = rexi:async_server_call({couch_server, Node}, {create, ShardName, Options}), - {Ref, {Node, Part}} + {Ref, Part} end, Fullmap). -%% @doc set up the receive loop with an overall timeout --spec create_db_loop([ref_node_part()]) -> {ok, np_acc()}. -create_db_loop(RefNodePart) -> - TimeoutRef = erlang:make_ref(), - {ok, TRef} = timer:send_after(5000, {timeout, TimeoutRef}), - Results = create_db_loop(RefNodePart, TimeoutRef, []), - timer:cancel(TRef), - Results. - %% @doc create_db receive loop %% Acc is either an accumulation of responses, or if we've received all %% responses, it's {ok, Responses} --spec create_db_loop([ref_node_part()], tref(), np_acc()) -> - np_acc() | {ok, np_acc()}. +-spec create_db_loop([ref_part_map()], tref(), beg_acc()) -> + beg_acc() | {ok, beg_acc()}. create_db_loop(_,_,{ok, Acc}) -> {ok, Acc}; -create_db_loop(RefNodePart, TimeoutRef, AccIn) -> +create_db_loop(RefPartMap, TimeoutRef, AccIn) -> receive {Ref, {ok, MainPid}} when is_reference(Ref) -> % for dev only, close the Fd TODO: remove me gen_server:call({couch_server, node(MainPid)}, {force_close, MainPid}), - AccOut = check_all_parts(Ref, RefNodePart, AccIn, ok), - create_db_loop(RefNodePart, TimeoutRef, AccOut); + AccOut = check_all_parts(Ref, RefPartMap, AccIn, ok), + create_db_loop(RefPartMap, TimeoutRef, AccOut); {Ref, Reply} when is_reference(Ref) -> - AccOut = check_all_parts(Ref, RefNodePart, AccIn, Reply), - create_db_loop(RefNodePart, TimeoutRef, AccOut); + AccOut = check_all_parts(Ref, RefPartMap, AccIn, Reply), + create_db_loop(RefPartMap, TimeoutRef, AccOut); {timeout, TimeoutRef} -> {error, timeout} end. -%% @doc check the results of the create replies -%% If we have a good reply from each partition, return ok --spec create_results(np_acc(), [ref_node_part()]) -> ok | create_quorum_error. -create_results(Results, RefNodePart) -> - ResultParts = create_result(Results, []), - DistinctParts = distinct_parts(RefNodePart), +%% @doc check the results (beginning of each partition range) of the create +%% replies. If we have a good reply from each partition, return ok +-spec create_results(beg_acc(), [ref_part_map()]) -> ok | create_quorum_error. +create_results(Results, RefPartMap) -> + ResultBegParts = create_result(Results, []), + DistinctBegParts = distinct_parts(RefPartMap), if - ResultParts =:= DistinctParts -> ok; + ResultBegParts =:= DistinctBegParts -> ok; true -> - ?debugFmt("~nResultParts: ~p~nDistinctParts: ~p~n", - [ResultParts, DistinctParts]), + ?debugFmt("~nResultBegParts: ~p~nDistinctBegParts: ~p~n", + [ResultBegParts, DistinctBegParts]), create_quorum_error end. --spec create_result(np_acc(), [np()]) -> [np()] | file_exists. +-spec create_result(beg_acc(), [part()]) -> [part()] | file_exists. create_result([], Acc) -> lists:usort(Acc); -create_result([{{_N,P}, ok}|Rest], Acc) -> - create_result(Rest, [P|Acc]); -create_result([{_NP, {error, file_exists}}|_Rest], _Acc) -> +create_result([{#part{b=Beg}, ok}|Rest], Acc) -> + create_result(Rest, [Beg|Acc]); +create_result([{_, {error, file_exists}}|_Rest], _Acc) -> {error, file_exists}; % if any replies were file_exists, return that -create_result([{{_N,_P}, Result}|Rest], Acc) -> +create_result([{_, Result}|Rest], Acc) -> showroom_log:message(error, "create_db error: ~p", [Result]), create_result(Rest, Acc). -check_all_parts(Ref, RefNodePart, Acc, Reply) -> - case couch_util:get_value(Ref, RefNodePart) of - {Node, Part} -> - case lists:keyfind({Node, Part}, 1, Acc) of +check_all_parts(Ref, RefPartMap, Acc, Reply) -> + case couch_util:get_value(Ref, RefPartMap) of + #part{} = Part -> + case lists:keyfind(Part, 1, Acc) of true -> Acc; % already present... that's odd _ -> - NewAcc = [{{Node, Part}, Reply} | Acc], - case length(NewAcc) >= length(RefNodePart) of + NewAcc = [{Part, Reply} | Acc], + case length(NewAcc) >= length(RefPartMap) of true -> {ok, NewAcc}; _ -> NewAcc end @@ -112,35 +104,7 @@ check_all_parts(Ref, RefNodePart, Acc, Reply) -> _ -> Acc % ignore a non-matching Ref end. -%% @doc check that we have a good reply from each partition. -%% If we do, return {ok, Acc}, if we don't, return Acc of partitions -%% Three 'case' statements and one 'if', a personal best. fml -%% @end -% check_distinct_parts(Ref, RefNodePart, Acc, Msg) -> -% Parts = distinct_parts(RefNodePart), -% case couch_util:get_value(Ref, RefNodePart) of -% {Node, Part} -> -% case lists:member(Part, Acc) of -% true -> Acc; -% _ -> -% case Msg of -% ok -> -% NewAcc = lists:usort([Part|Acc]), -% if -% Parts =:= NewAcc -> {ok, NewAcc}; -% true -> NewAcc -% end; -% _ -> -% Hex = showroom_utils:int_to_hexstr(Part), -% showroom_log:message(error, -% "create_db reply error: ~p from ~p ~p", [Msg, Node, Hex]), -% Acc -% end -% end; -% _ -> Acc % ignore a non-matching Ref -% end. - -distinct_parts(RefNodePart) -> - {_Refs, NPs} = lists:unzip(RefNodePart), - {_Nodes, Parts} = lists:unzip(NPs), - lists:usort(Parts). +distinct_parts(RefPartMap) -> + {_Refs, Parts} = lists:unzip(RefPartMap), + BegParts = lists:map(fun(#part{b=Beg}) -> Beg end, Parts), + lists:usort(BegParts). diff --git a/src/fabric_info.erl b/src/fabric_info.erl index 95408b6f..51529da3 100644 --- a/src/fabric_info.erl +++ b/src/fabric_info.erl @@ -1,23 +1,113 @@ -module(fabric_info). --export([all_databases/1]). +-export([all_databases/1, get_db_info/2]). -include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). %% @doc gets all databases in the cluster. -spec all_databases(binary() | []) -> [binary()]. all_databases([]) -> - Dbs = ets:foldl(fun({DbName, _}, Acc0) -> - [DbName | Acc0] - end, [], dbs_cache), + Dbs = ets:foldl(fun(#part{dbname=DbName}, AccIn) -> + new_acc(DbName, AccIn) + end, [], partitions), {ok, Dbs}; all_databases(Customer) -> ?debugFmt("~nCustomer: ~p~n", [Customer]), - Dbs = ets:foldl(fun({DbName, _}, Acc0) -> + Dbs = ets:foldl(fun(#part{dbname=DbName}, AccIn) -> DbNameStr = ?b2l(DbName), case string:str(DbNameStr, Customer) of - 1 -> [DbNameStr | Acc0]; - _ -> Acc0 + 1 -> + new_acc(DbNameStr, AccIn); + _ -> AccIn end end, [], dbs_cache), {ok, Dbs}. + +%% @doc get database information tuple +get_db_info(DbName, _Customer) -> + Parts = partitions:all_parts(DbName), + RefPartMap = send_info_calls(DbName, Parts), + {ok, Results} = fabric_rpc:receive_loop(RefPartMap, 5000, fun info_loop/3), + InfoList = Results, + % process_infos(ShardInfos, [{db_name, Name}]); + {ok, InfoList}. + +%% ===================== +%% internal +%% ===================== + +new_acc(DbName, Acc) -> + case lists:member(DbName, Acc) of + true -> Acc; + _ ->[DbName | Acc] + end. + +send_info_calls(DbName, Parts) -> + lists:map(fun(#part{node=Node, b=Beg} = Part) -> + ShardName = showroom_utils:shard_name(Beg, DbName), + Ref = rexi:cast(Node, {rexi_rpc, get_db_info, ShardName}), + {Ref, Part} + end, Parts). + +%% @doc create_db receive loop +%% Acc is either an accumulation of responses, or if we've received all +%% responses, it's {ok, Responses} +-spec info_loop([ref_part_map()], tref(), beg_acc()) -> + beg_acc() | {ok, beg_acc()}. +info_loop(_,_,{ok, Acc}) -> {ok, Acc}; +info_loop(RefPartMap, TimeoutRef, AccIn) -> + receive + {Ref, {ok, Info}} when is_reference(Ref) -> + AccOut = check_all_parts(Ref, RefPartMap, AccIn, ok), + info_loop(RefPartMap, TimeoutRef, AccOut); + {Ref, Reply} when is_reference(Ref) -> + AccOut = check_all_parts(Ref, RefPartMap, AccIn, Reply), + info_loop(RefPartMap, TimeoutRef, AccOut); + {timeout, TimeoutRef} -> + {error, timeout} + end. + + +cloudant_db_name(Customer, FullName) -> + case Customer of + "" -> + FullName; + Name -> + re:replace(FullName, [Name,"/"], "", [{return, binary}]) + end. + +%% Loop through Tasks on the flattened Infos and get the aggregated result +process_infos(Infos, Initial) -> + Tasks = [ + {doc_count, fun sum/2, 0}, + {doc_del_count, fun sum/2, 0}, + {update_seq, fun max/2, 1}, + {purge_seq, fun sum/2, 0}, + {compact_running, fun bool/2, 0}, + {disk_size, fun sum/2, 0}, + {instance_start_time, fun(_, _) -> <<"0">> end, 0}, + {disk_format_version, fun max/2, 0}], + + Infos1 = lists:flatten(Infos), + + Result = lists:map(fun({Type, Fun, Default}) -> + {Type, process_info(Type, Fun, Default, Infos1)} + end, Tasks), + lists:flatten([Initial, Result]). + + process_info(Type, Fun, Default, List) -> + lists:foldl(fun(V, AccIn) -> Fun(V, AccIn) end, Default, + couch_util:get_all_values(Type, List)). + +sum(New, Existing) -> + New + Existing. + +bool(New, Existing) -> + New andalso Existing. + +max(New, Existing) -> + case New > Existing of + true -> New; + false -> Existing + end. diff --git a/src/fabric_open.erl b/src/fabric_open.erl index a8c0204e..289a0681 100644 --- a/src/fabric_open.erl +++ b/src/fabric_open.erl @@ -23,25 +23,12 @@ % open_db(<<"S", ShardName/binary>>, Options). -open_doc(DbName, DocId, Revs, Options) -> +open_doc(DbName, DocId, _Revs, _Options) -> NPs = partitions:key_nodes_parts(DbName, DocId), ?debugFmt("~nNPs: ~p~n", [NPs]), - ok. - - -open_doc_endpoint(DbName, DocId, Revs, Options) -> - case couch_db:open(DbName, []) of - {ok, Db} -> - try - couch_api:open_doc(Db, DocId, Revs, Options) - after - couch_db:close(Db) - end; - {not_found, no_db_file} -> - throw({not_found, <<"The database does not exist.">>}); - Error -> - throw(Error) - end. + {ok, #doc{}}. + + -- cgit v1.2.3 From 47bd7a3293bb41c9d040e7fd35dcfe9faf16a992 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Wed, 26 May 2010 18:36:44 -0400 Subject: rework create_db with new idioms for fabric / rexi workflow --- ebin/fabric.app | 3 +- src/fabric_create.erl | 98 ++++++++++++++++----------------------------------- src/fabric_rpc.erl | 49 ++++++++------------------ src/fabric_util.erl | 49 ++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 103 deletions(-) create mode 100644 src/fabric_util.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index 73943af5..ba3d3e03 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -9,7 +9,8 @@ fabric_create, fabric_delete, fabric_info, - fabric_open + fabric_open, + fabric_util ]}, {registered, []}, {included_applications, []}, diff --git a/src/fabric_create.erl b/src/fabric_create.erl index 1143d080..c2c01ccd 100644 --- a/src/fabric_create.erl +++ b/src/fabric_create.erl @@ -19,13 +19,14 @@ create_db(DbName, Options) -> Fullmap = partitions:fullmap(DbName, Options), {ok, FullNodes} = mem3:fullnodes(), RefPartMap = send_create_calls(DbName, Options, Fullmap), - {ok, Results} = fabric_rpc:receive_loop(RefPartMap, 5000, - fun create_db_loop/3), - case create_results(Results, RefPartMap) of - ok -> + Acc0 = {false, length(RefPartMap), + lists:usort([ {Beg, false} || {_,#part{b=Beg}} <- RefPartMap])}, + case fabric_util:receive_loop( + RefPartMap, 1, fun handle_create_msg/3, Acc0, 5000, infinity) of + {ok, _Results} -> partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), - {ok, #db{name=DbName}}; - Other -> {error, Other} + ok; + Error -> Error end. @@ -43,68 +44,29 @@ send_create_calls(DbName, Options, Fullmap) -> {Ref, Part} end, Fullmap). -%% @doc create_db receive loop -%% Acc is either an accumulation of responses, or if we've received all -%% responses, it's {ok, Responses} --spec create_db_loop([ref_part_map()], tref(), beg_acc()) -> - beg_acc() | {ok, beg_acc()}. -create_db_loop(_,_,{ok, Acc}) -> {ok, Acc}; -create_db_loop(RefPartMap, TimeoutRef, AccIn) -> - receive - {Ref, {ok, MainPid}} when is_reference(Ref) -> - % for dev only, close the Fd TODO: remove me - gen_server:call({couch_server, node(MainPid)}, {force_close, MainPid}), - - AccOut = check_all_parts(Ref, RefPartMap, AccIn, ok), - create_db_loop(RefPartMap, TimeoutRef, AccOut); - {Ref, Reply} when is_reference(Ref) -> - AccOut = check_all_parts(Ref, RefPartMap, AccIn, Reply), - create_db_loop(RefPartMap, TimeoutRef, AccOut); - {timeout, TimeoutRef} -> - {error, timeout} - end. - -%% @doc check the results (beginning of each partition range) of the create -%% replies. If we have a good reply from each partition, return ok --spec create_results(beg_acc(), [ref_part_map()]) -> ok | create_quorum_error. -create_results(Results, RefPartMap) -> - ResultBegParts = create_result(Results, []), - DistinctBegParts = distinct_parts(RefPartMap), +handle_create_msg(_, file_exists, _) -> + {error, file_exists}; +handle_create_msg(_, {rexi_EXIT, _Reason}, {Complete, N, Parts}) -> + {ok, {Complete, N-1, Parts}}; +handle_create_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> if - ResultBegParts =:= DistinctBegParts -> ok; - true -> - ?debugFmt("~nResultBegParts: ~p~nDistinctBegParts: ~p~n", - [ResultBegParts, DistinctBegParts]), - create_quorum_error - end. - --spec create_result(beg_acc(), [part()]) -> [part()] | file_exists. -create_result([], Acc) -> - lists:usort(Acc); -create_result([{#part{b=Beg}, ok}|Rest], Acc) -> - create_result(Rest, [Beg|Acc]); -create_result([{_, {error, file_exists}}|_Rest], _Acc) -> - {error, file_exists}; % if any replies were file_exists, return that -create_result([{_, Result}|Rest], Acc) -> - showroom_log:message(error, "create_db error: ~p", [Result]), - create_result(Rest, Acc). + Complete -> {stop, ok}; + true -> {error, create_db_fubar} + end; +handle_create_msg(_, _, {true, 1, _Acc}) -> + {stop, ok}; +handle_create_msg({_, #part{b=Beg}}, {ok, _}, {false, 1, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + case is_complete(PartResults) of + true -> {stop, ok}; + false -> {error, create_db_fubar} + end; +handle_create_msg(_RefPart, {ok, _}, {true, N, Parts}) -> + {ok, {true, N-1, Parts}}; +handle_create_msg({_Ref, #part{b=Beg}}, {ok, _}, {false, Rem, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + {ok, {is_complete(PartResults), Rem-1, PartResults}}. -check_all_parts(Ref, RefPartMap, Acc, Reply) -> - case couch_util:get_value(Ref, RefPartMap) of - #part{} = Part -> - case lists:keyfind(Part, 1, Acc) of - true -> Acc; % already present... that's odd - _ -> - NewAcc = [{Part, Reply} | Acc], - case length(NewAcc) >= length(RefPartMap) of - true -> {ok, NewAcc}; - _ -> NewAcc - end - end; - _ -> Acc % ignore a non-matching Ref - end. -distinct_parts(RefPartMap) -> - {_Refs, Parts} = lists:unzip(RefPartMap), - BegParts = lists:map(fun(#part{b=Beg}) -> Beg end, Parts), - lists:usort(BegParts). +is_complete(List) -> + lists:all(fun({_,Bool}) -> Bool end, List). diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index fd54827c..0a034d59 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -1,49 +1,30 @@ -module(fabric_rpc). --export([open_doc/4, get_doc_info/1]). --export([receive_loop/3]). +-export([open_doc/4, get_db_info/1]). --include("../../dynomite/include/membership.hrl"). +%% rpc endpoints +%% call to with_db will supply your M:F with a #db{} and then remaining args open_doc(DbName, DocId, Revs, Options) -> - case couch_db:open(DbName, []) of - {ok, Db} -> - try - couch_api:open_doc(Db, DocId, Revs, Options) - after - couch_db:close(Db) - end; - {not_found, no_db_file} -> - throw({not_found, <<"The database does not exist.">>}); - Error -> - throw(Error) - end. + with_db(DbName, {couch_api, open_doc, [DocId, Revs, Options]}). + +get_db_info(DbName) -> + with_db(DbName, {couch_db, get_db_info, []}). -get_doc_info(DbName) -> +%% +%% internal +%% + +with_db(DbName, {M,F,A}) -> case couch_db:open(DbName, []) of {ok, Db} -> - try - couch_db:get_db_info(Db) - after - couch_db:close(Db) - end; - {not_found, no_db_file} -> - throw({not_found, <<"The database does not exist.">>}); + rexi:reply(apply(M, F, [Db | A])); Error -> - throw(Error) + rexi:reply(Error) end. + %% %% helper funs %% - -%% @doc set up the receive loop with an overall timeout --spec receive_loop([ref_part_map()], integer(), function()) -> {ok, beg_acc()}. -receive_loop(RefPartMap, Timeout, Loop) -> - TimeoutRef = erlang:make_ref(), - {ok, TRef} = timer:send_after(Timeout, {timeout, TimeoutRef}), - Results = Loop(RefPartMap, TimeoutRef, []), - timer:cancel(TRef), - Results. -%{Status, Info} = couch_db:get_db_info(Shard), diff --git a/src/fabric_util.erl b/src/fabric_util.erl new file mode 100644 index 00000000..8eeaee33 --- /dev/null +++ b/src/fabric_util.erl @@ -0,0 +1,49 @@ +-module(fabric_util). + +-export([receive_loop/6]). + +-include("../../dynomite/include/membership.hrl"). + + +%% @doc set up the receive loop with an overall timeout +-spec receive_loop([ref_part_map()], integer(), function(), any(), + integer(), integer()) -> + {ok, beg_acc()}. +receive_loop(RefPartMap, Keypos, Fun, Acc0, GlobalTimeout, PerMsgTO) -> + TimeoutRef = erlang:make_ref(), + {ok, TRef} = timer:send_after(GlobalTimeout, {timeout, TimeoutRef}), + try + process_mailbox(RefPartMap, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) + after + timer:cancel(TRef) + end. + +process_mailbox(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> + case process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) of + {ok, Acc} -> + process_mailbox(RefList, Keypos, Fun, Acc, TimeoutRef, PerMsgTO); + {stop, Acc} -> + {ok, Acc}; + Error -> + Error + end. + +process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> + receive + {timeout, TimeoutRef} -> + timeout; + {Ref, Msg} -> + case lists:keyfind(Ref, Keypos, RefList) of + false -> + % this was some non-matching message which we will ignore + {ok, Acc0}; + RefPart -> + % call the Fun that understands the message + Fun(RefPart, Msg, Acc0) + end; + {rexi_DOWN, _RexiMonPid, ServerPid, Reason} = Msg -> + showroom_log:message(alert, "rexi_DOWN ~p ~p", [ServerPid, Reason]), + Fun(nil, Msg, Acc0) + after PerMsgTO -> + timeout + end. -- cgit v1.2.3 From 41a927b8c2c64eaa56c7f1fe1bb250021004c924 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 27 May 2010 10:52:05 -0400 Subject: changes to get compile working again, and testing create_db --- ebin/fabric.app | 1 + src/fabric_info.erl | 10 ++++++---- src/fabric_open.erl | 1 - 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index ba3d3e03..7e425a03 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -10,6 +10,7 @@ fabric_delete, fabric_info, fabric_open, + fabric_rpc, fabric_util ]}, {registered, []}, diff --git a/src/fabric_info.erl b/src/fabric_info.erl index 51529da3..90cd11f0 100644 --- a/src/fabric_info.erl +++ b/src/fabric_info.erl @@ -59,11 +59,13 @@ info_loop(_,_,{ok, Acc}) -> {ok, Acc}; info_loop(RefPartMap, TimeoutRef, AccIn) -> receive {Ref, {ok, Info}} when is_reference(Ref) -> - AccOut = check_all_parts(Ref, RefPartMap, AccIn, ok), - info_loop(RefPartMap, TimeoutRef, AccOut); + %AccOut = check_all_parts(Ref, RefPartMap, AccIn, ok), + %info_loop(RefPartMap, TimeoutRef, AccOut); + ok; {Ref, Reply} when is_reference(Ref) -> - AccOut = check_all_parts(Ref, RefPartMap, AccIn, Reply), - info_loop(RefPartMap, TimeoutRef, AccOut); + %AccOut = check_all_parts(Ref, RefPartMap, AccIn, Reply), + %info_loop(RefPartMap, TimeoutRef, AccOut); + ok; {timeout, TimeoutRef} -> {error, timeout} end. diff --git a/src/fabric_open.erl b/src/fabric_open.erl index 289a0681..cab10c5d 100644 --- a/src/fabric_open.erl +++ b/src/fabric_open.erl @@ -1,7 +1,6 @@ -module(fabric_open). -export([open_doc/4]). --export([open_doc_endpoint/4]). -include("../../couch/src/couch_db.hrl"). -- cgit v1.2.3 From 00276b2582be213adf46c1f2333a65ceb86809e2 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 27 May 2010 10:54:31 -0400 Subject: begin move of delete_db over to new codebase --- src/fabric_delete.erl | 169 +++++++++++++++----------------------------------- 1 file changed, 49 insertions(+), 120 deletions(-) diff --git a/src/fabric_delete.erl b/src/fabric_delete.erl index e77f813f..d1148e40 100644 --- a/src/fabric_delete.erl +++ b/src/fabric_delete.erl @@ -12,137 +12,66 @@ %% api %% ===================== -%% @doc Delete a new database, and all its partition files across the cluster +%% @doc Delete a database, and all its partition files across the cluster %% Options is proplist with user_ctx, n, q -spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. delete_db(DbName, Options) -> - Fullmap = partitions:fullmap(DbName, Options), - RefNodePart = send_delete_calls(DbName, Options, Fullmap), - {ok, Results} = delete_db_loop(RefNodePart), - delete_results(Results, RefNodePart). - + Parts = partitions:all_parts(DbName), + RefPartMap = send_calls(DbName, Options, Parts), + Acc0 = {false, length(RefPartMap)}, + case fabric_util:receive_loop( + RefPartMap, 1, fun handle_delete_msg/3, Acc0, 5000, infinity) of + {ok, _Results} -> + delete_fullmap(DbName), + ok; + Error -> Error + end. -%delete_db(DbName, Options) -> -% ResolveFun = fun(_Good) -> true end, -% case cluster_ops:all_parts({dynomite_couch_api,delete_db,[DbName, Options]}, -% d, true, ResolveFun) of -% {ok, true} -> ok; -% [{error, d_quorum_not_met}, {good, _Good}, {bad, Bad}] -> -% showroom_utils:first_bad(Bad); -% [{error, Error}, {good, _Good}, {bad, Bad}] -> -% {Error, showroom_utils:first_bad(Bad)}; -% Other -> -% ?debugFmt("~nOther: ~p~n", [Other]), -% Other -% end. %% ===================== %% internal %% ===================== %% @doc delete the partitions on all appropriate nodes (rexi calls) --spec send_delete_calls(binary(), list(), [mem_node()]) -> [{reference(), np()}]. -send_delete_calls(DbName, Options, Fullmap) -> - lists:map(fun({Node, Part}) -> - ShardName = showroom_utils:shard_name(Part, DbName), +-spec send_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. +send_calls(DbName, Options, Parts) -> + lists:map(fun(#part{node=Node, b=Beg} = Part) -> + ShardName = showroom_utils:shard_name(Beg, DbName), Ref = rexi:async_server_call({couch_server, Node}, {delete, ShardName, Options}), - {Ref, {Node, Part}} - end, Fullmap). - -%% @doc set up the receive loop with an overall timeout --spec delete_db_loop([ref_node_part()]) -> {ok, np_acc()}. -delete_db_loop(RefNodePart) -> - TimeoutRef = erlang:make_ref(), - {ok, TRef} = timer:send_after(5000, {timeout, TimeoutRef}), - Results = delete_db_loop(RefNodePart, TimeoutRef, []), - timer:cancel(TRef), - Results. - -%% @doc delete_db receive loop -%% Acc is either an accumulation of responses, or if we've received all -%% responses, it's {ok, Responses} --spec delete_db_loop([ref_node_part()], tref(), np_acc()) -> - np_acc() | {ok, np_acc()}. -delete_db_loop(_,_,{ok, Acc}) -> {ok, Acc}; -delete_db_loop(RefNodePart, TimeoutRef, AccIn) -> - receive - {Ref, {ok, deleted}} when is_reference(Ref) -> - AccOut = check_all_parts(Ref, RefNodePart, AccIn, ok), - delete_db_loop(RefNodePart, TimeoutRef, AccOut); - {Ref, Reply} when is_reference(Ref) -> - AccOut = check_all_parts(Ref, RefNodePart, AccIn, Reply), - delete_db_loop(RefNodePart, TimeoutRef, AccOut); - {timeout, TimeoutRef} -> - {error, timeout} - end. - -%% @doc check the results of the delete replies -%% If we have a good reply from all partitions, return ok --spec delete_results(np_acc(), [ref_node_part()]) -> - ok | {error, delete_quorum_error}. -delete_results(Results, RefNodePart) -> - ResultNPs = delete_result(Results, []), - AllNPs = all_nodes_parts(RefNodePart), + {Ref, Part} + end, Parts). + +handle_delete_msg(_, not_found, _) -> + {error, not_found}; +handle_delete_msg(_, {rexi_EXIT, _Reason}, {Complete, N, Parts}) -> + {ok, {Complete, N-1, Parts}}; +handle_delete_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> if - ResultNPs =:= AllNPs -> ok; - true -> {error, delete_quorum_error} + Complete -> {stop, ok}; + true -> {error, delete_db_fubar} + end; +handle_delete_msg(_, _, {true, 1, _Acc}) -> + {stop, ok}; +handle_delete_msg({_, #part{b=Beg}}, {ok, _}, {false, 1, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + case is_complete(PartResults) of + true -> {stop, ok}; + false -> {error, delete_db_fubar} + end; +handle_delete_msg(_RefPart, {ok, _}, {true, N, Parts}) -> + {ok, {true, N-1, Parts}}; +handle_delete_msg({_Ref, #part{b=Beg}}, {ok, _}, {false, Rem, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + {ok, {is_complete(PartResults), Rem-1, PartResults}}. + +is_complete(List) -> + lists:all(fun({_,Bool}) -> Bool end, List). + +delete_fullmap(DbName) -> + case couch_db:open(<<"dbs">>, []) of + {ok, Db} -> + couch_api:open_doc(Db, DbName, nil, []), + couch_api:update_doc(Db, DbName, {[{<<"_deleted">>,true}]}); + Error -> Error end. - --spec delete_result(np_acc(), [np()]) -> [np()] | file_exists. -delete_result([], Acc) -> - lists:sort(Acc); -delete_result([{NP, ok}|Rest], Acc) -> - delete_result(Rest, [NP|Acc]); -delete_result([{_NP, {error, file_exists}}|_Rest], _Acc) -> - {error, file_exists}; % if any replies were file_exists, return that -delete_result([{{_N,_P}, Result}|Rest], Acc) -> - ?LOG_ERROR("delete_db error: ~p", [Result]), - delete_result(Rest, Acc). - -check_all_parts(Ref, RefNodePart, Acc, Reply) -> - case couch_util:get_value(Ref, RefNodePart) of - {Node, Part} -> - case lists:keyfind({Node, Part}, 1, Acc) of - true -> Acc; % already present... that's odd - _ -> - NewAcc = [{{Node, Part}, Reply} | Acc], - case length(NewAcc) >= length(RefNodePart) of - true -> {ok, NewAcc}; - _ -> NewAcc - end - end; - _ -> Acc % ignore a non-matching Ref - end. - -%% @doc check that we have a good reply from each partition. -%% If we do, return {ok, Acc}, if we don't, return Acc of partitions -%% Three 'case' statements and one 'if', a personal best. fml -%% @end -% check_distinct_parts(Ref, RefNodePart, Acc, Msg) -> -% Parts = distinct_parts(RefNodePart), -% case couch_util:get_value(Ref, RefNodePart) of -% {Node, Part} -> -% case lists:member(Part, Acc) of -% true -> Acc; -% _ -> -% case Msg of -% ok -> -% NewAcc = lists:usort([Part|Acc]), -% if -% Parts =:= NewAcc -> {ok, NewAcc}; -% true -> NewAcc -% end; -% _ -> -% Hex = showroom_utils:int_to_hexstr(Part), -% showroom_log:message(error, -% "delete_db reply error: ~p from ~p ~p", [Msg, Node, Hex]), -% Acc -% end -% end; -% _ -> Acc % ignore a non-matching Ref -% end. - -all_nodes_parts(RefNodePart) -> - {_Refs, NPs} = lists:unzip(RefNodePart), - lists:sort(NPs). -- cgit v1.2.3 From f399485a90f5d09f6b2df839511756642352031b Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 27 May 2010 10:52:51 -0400 Subject: replace #part with #shard --- src/fabric_create.erl | 8 ++++---- src/fabric_delete.erl | 6 +++--- src/fabric_info.erl | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/fabric_create.erl b/src/fabric_create.erl index c2c01ccd..17c85b6a 100644 --- a/src/fabric_create.erl +++ b/src/fabric_create.erl @@ -20,7 +20,7 @@ create_db(DbName, Options) -> {ok, FullNodes} = mem3:fullnodes(), RefPartMap = send_create_calls(DbName, Options, Fullmap), Acc0 = {false, length(RefPartMap), - lists:usort([ {Beg, false} || {_,#part{b=Beg}} <- RefPartMap])}, + lists:usort([ {Beg, false} || {_,#shard{range=[Beg,_]}} <- RefPartMap])}, case fabric_util:receive_loop( RefPartMap, 1, fun handle_create_msg/3, Acc0, 5000, infinity) of {ok, _Results} -> @@ -37,7 +37,7 @@ create_db(DbName, Options) -> %% @doc create the partitions on all appropriate nodes (rexi calls) -spec send_create_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. send_create_calls(DbName, Options, Fullmap) -> - lists:map(fun(#part{node=Node, b=Beg} = Part) -> + lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> ShardName = showroom_utils:shard_name(Beg, DbName), Ref = rexi:async_server_call({couch_server, Node}, {create, ShardName, Options}), @@ -55,7 +55,7 @@ handle_create_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> end; handle_create_msg(_, _, {true, 1, _Acc}) -> {stop, ok}; -handle_create_msg({_, #part{b=Beg}}, {ok, _}, {false, 1, PartResults0}) -> +handle_create_msg({_, #shard{range=[Beg,_]}}, {ok, _}, {false, 1, PartResults0}) -> PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), case is_complete(PartResults) of true -> {stop, ok}; @@ -63,7 +63,7 @@ handle_create_msg({_, #part{b=Beg}}, {ok, _}, {false, 1, PartResults0}) -> end; handle_create_msg(_RefPart, {ok, _}, {true, N, Parts}) -> {ok, {true, N-1, Parts}}; -handle_create_msg({_Ref, #part{b=Beg}}, {ok, _}, {false, Rem, PartResults0}) -> +handle_create_msg({_Ref, #shard{range=[Beg,_]}}, {ok, _}, {false, Rem, PartResults0}) -> PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), {ok, {is_complete(PartResults), Rem-1, PartResults}}. diff --git a/src/fabric_delete.erl b/src/fabric_delete.erl index d1148e40..90108d3a 100644 --- a/src/fabric_delete.erl +++ b/src/fabric_delete.erl @@ -35,7 +35,7 @@ delete_db(DbName, Options) -> %% @doc delete the partitions on all appropriate nodes (rexi calls) -spec send_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. send_calls(DbName, Options, Parts) -> - lists:map(fun(#part{node=Node, b=Beg} = Part) -> + lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> ShardName = showroom_utils:shard_name(Beg, DbName), Ref = rexi:async_server_call({couch_server, Node}, {delete, ShardName, Options}), @@ -53,7 +53,7 @@ handle_delete_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> end; handle_delete_msg(_, _, {true, 1, _Acc}) -> {stop, ok}; -handle_delete_msg({_, #part{b=Beg}}, {ok, _}, {false, 1, PartResults0}) -> +handle_delete_msg({_, #shard{range=[Beg,_]}}, {ok, _}, {false, 1, PartResults0}) -> PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), case is_complete(PartResults) of true -> {stop, ok}; @@ -61,7 +61,7 @@ handle_delete_msg({_, #part{b=Beg}}, {ok, _}, {false, 1, PartResults0}) -> end; handle_delete_msg(_RefPart, {ok, _}, {true, N, Parts}) -> {ok, {true, N-1, Parts}}; -handle_delete_msg({_Ref, #part{b=Beg}}, {ok, _}, {false, Rem, PartResults0}) -> +handle_delete_msg({_Ref, #shard{range=[Beg,_]}}, {ok, _}, {false, Rem, PartResults0}) -> PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), {ok, {is_complete(PartResults), Rem-1, PartResults}}. diff --git a/src/fabric_info.erl b/src/fabric_info.erl index 90cd11f0..662c06b6 100644 --- a/src/fabric_info.erl +++ b/src/fabric_info.erl @@ -8,13 +8,13 @@ %% @doc gets all databases in the cluster. -spec all_databases(binary() | []) -> [binary()]. all_databases([]) -> - Dbs = ets:foldl(fun(#part{dbname=DbName}, AccIn) -> + Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> new_acc(DbName, AccIn) end, [], partitions), {ok, Dbs}; all_databases(Customer) -> ?debugFmt("~nCustomer: ~p~n", [Customer]), - Dbs = ets:foldl(fun(#part{dbname=DbName}, AccIn) -> + Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> DbNameStr = ?b2l(DbName), case string:str(DbNameStr, Customer) of 1 -> @@ -44,7 +44,7 @@ new_acc(DbName, Acc) -> end. send_info_calls(DbName, Parts) -> - lists:map(fun(#part{node=Node, b=Beg} = Part) -> + lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> ShardName = showroom_utils:shard_name(Beg, DbName), Ref = rexi:cast(Node, {rexi_rpc, get_db_info, ShardName}), {Ref, Part} -- cgit v1.2.3 From 5b68d7349807b6540514ac8b506e80ede3d2fe3d Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 27 May 2010 11:37:11 -0400 Subject: more delete_db changes --- src/fabric_delete.erl | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/src/fabric_delete.erl b/src/fabric_delete.erl index d1148e40..d2ee47e8 100644 --- a/src/fabric_delete.erl +++ b/src/fabric_delete.erl @@ -18,7 +18,7 @@ delete_db(DbName, Options) -> Parts = partitions:all_parts(DbName), RefPartMap = send_calls(DbName, Options, Parts), - Acc0 = {false, length(RefPartMap)}, + Acc0 = {true, length(RefPartMap)}, case fabric_util:receive_loop( RefPartMap, 1, fun handle_delete_msg/3, Acc0, 5000, infinity) of {ok, _Results} -> @@ -42,31 +42,19 @@ send_calls(DbName, Options, Parts) -> {Ref, Part} end, Parts). -handle_delete_msg(_, not_found, _) -> - {error, not_found}; -handle_delete_msg(_, {rexi_EXIT, _Reason}, {Complete, N, Parts}) -> - {ok, {Complete, N-1, Parts}}; -handle_delete_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> +handle_delete_msg(_, not_found, {NotFound, N}) -> + {ok, {NotFound, N-1}}; +handle_delete_msg(_, {rexi_EXIT, _Reason}, {NotFound, N}) -> + {ok, {NotFound, N-1}}; +handle_delete_msg(_, {rexi_DOWN, _, _, _}, _Acc) -> + {error, delete_db_fubar}; +handle_delete_msg(_, _, {NotFound, 1}) -> if - Complete -> {stop, ok}; - true -> {error, delete_db_fubar} + NotFound -> {stop, not_found}; + true -> {stop, ok} end; -handle_delete_msg(_, _, {true, 1, _Acc}) -> - {stop, ok}; -handle_delete_msg({_, #part{b=Beg}}, {ok, _}, {false, 1, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - case is_complete(PartResults) of - true -> {stop, ok}; - false -> {error, delete_db_fubar} - end; -handle_delete_msg(_RefPart, {ok, _}, {true, N, Parts}) -> - {ok, {true, N-1, Parts}}; -handle_delete_msg({_Ref, #part{b=Beg}}, {ok, _}, {false, Rem, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - {ok, {is_complete(PartResults), Rem-1, PartResults}}. - -is_complete(List) -> - lists:all(fun({_,Bool}) -> Bool end, List). +handle_delete_msg(_, ok, {_NotFound, N}) -> + {ok, {false, N-1}}. delete_fullmap(DbName) -> case couch_db:open(<<"dbs">>, []) of -- cgit v1.2.3 From cad5e1f1ef75c2338171916b7cb76549a2f6ffad Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 27 May 2010 12:41:31 -0400 Subject: delete_db working --- src/fabric_delete.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabric_delete.erl b/src/fabric_delete.erl index 674f09ad..9a65f72b 100644 --- a/src/fabric_delete.erl +++ b/src/fabric_delete.erl @@ -59,7 +59,7 @@ handle_delete_msg(_, ok, {_NotFound, N}) -> delete_fullmap(DbName) -> case couch_db:open(<<"dbs">>, []) of {ok, Db} -> - couch_api:open_doc(Db, DbName, nil, []), - couch_api:update_doc(Db, DbName, {[{<<"_deleted">>,true}]}); + {ok, Doc} = couch_api:open_doc(Db, DbName, nil, []), + couch_api:update_doc(Db, Doc#doc{deleted=true}); Error -> Error end. -- cgit v1.2.3 From 200d64d61d209599a1f32b954605c46abacd45bc Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 27 May 2010 12:46:15 -0400 Subject: open_doc call in fabric --- ebin/fabric.app | 3 ++- src/fabric_util.erl | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index 7e425a03..cd81b383 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -11,7 +11,8 @@ fabric_info, fabric_open, fabric_rpc, - fabric_util + fabric_util, + fabric_doc ]}, {registered, []}, {included_applications, []}, diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 8eeaee33..98da1001 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -1,14 +1,30 @@ -module(fabric_util). --export([receive_loop/6]). +-export([receive_loop/4, receive_loop/6]). -include("../../dynomite/include/membership.hrl"). +submit_jobs(Shards, EndPoint, ExtraArgs) -> + lists:map(fun(#shard{node=Node, name=ShardName} = Shard) -> + Ref = rexi:cast(Node, {?RPC, EndPoint, [ShardName | ExtraArgs]}), + Shard#shard{ref = Ref} + end. + +recv(Workers, Keypos, Fun, Acc0) -> + receive_loop(Workers, Keypos, Fun, Acc0). + +receive_loop(Workers, Keypos, Fun, Acc0) -> + case couch_config:get("fabric", "request_timeout", "10000") of + "infinity" -> + Timeout = infinity; + N -> + Timeout = list_to_integer(N) + end, + receive_loop(Workers, Keypos, Fun, Acc0, Timeout, infinity). %% @doc set up the receive loop with an overall timeout --spec receive_loop([ref_part_map()], integer(), function(), any(), - integer(), integer()) -> - {ok, beg_acc()}. +-spec receive_loop([any()], integer(), function(), any(), timeout(), timeout()) -> + {ok, any()}. receive_loop(RefPartMap, Keypos, Fun, Acc0, GlobalTimeout, PerMsgTO) -> TimeoutRef = erlang:make_ref(), {ok, TRef} = timer:send_after(GlobalTimeout, {timeout, TimeoutRef}), -- cgit v1.2.3 From e91bbf3d70eefdd44c123e06c12e0b9ab498a791 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 27 May 2010 12:51:11 -0400 Subject: sloppy commit --- src/fabric_util.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 98da1001..4b2f611a 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -1,14 +1,14 @@ -module(fabric_util). --export([receive_loop/4, receive_loop/6]). +-export([submit_jobs/3, recv/4, receive_loop/4, receive_loop/6]). -include("../../dynomite/include/membership.hrl"). submit_jobs(Shards, EndPoint, ExtraArgs) -> lists:map(fun(#shard{node=Node, name=ShardName} = Shard) -> - Ref = rexi:cast(Node, {?RPC, EndPoint, [ShardName | ExtraArgs]}), + Ref = rexi:cast(Node, {fabric_rpc, EndPoint, [ShardName | ExtraArgs]}), Shard#shard{ref = Ref} - end. + end, Shards). recv(Workers, Keypos, Fun, Acc0) -> receive_loop(Workers, Keypos, Fun, Acc0). -- cgit v1.2.3 From 408cf4c2f0d7fcf0fa35e4b9e28a8ac86c5c4fcd Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 27 May 2010 15:45:08 -0400 Subject: export the right stuff for open_doc --- src/fabric.erl | 6 +++--- src/fabric_rpc.erl | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index a54264dc..15269bbb 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,6 +1,6 @@ -module(fabric). --export([all_databases/1, create_db/2, delete_db/2, open_db/2, open_doc/4, +-export([all_databases/1, create_db/2, delete_db/2, open_doc/3, open_doc/4, get_db_info/2]). %% maybe this goes away, and these are called directly in their own modules in @@ -15,8 +15,8 @@ create_db(DbName, Options) -> delete_db(DbName, Options) -> fabric_delete:delete_db(DbName, Options). -open_db(DbName, Options) -> - fabric_open:open_db(DbName, Options). +open_doc(Db, DocId, Options) -> + fabric_doc:open_doc(Db, DocId, Options). open_doc(Db, DocId, Revs, Options) -> fabric_open:open_doc(Db, DocId, Revs, Options). diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 0a034d59..49809145 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -1,10 +1,9 @@ -module(fabric_rpc). --export([open_doc/4, get_db_info/1]). +-export([open_doc/3, open_doc/4, get_db_info/1]). - -%% rpc endpoints -%% call to with_db will supply your M:F with a #db{} and then remaining args +open_doc(DbName, DocId, Options) -> + with_db(DbName, {couch_db, open_doc, [DocId, Options]}). open_doc(DbName, DocId, Revs, Options) -> with_db(DbName, {couch_api, open_doc, [DocId, Revs, Options]}). -- cgit v1.2.3 From 6cba6c59340b1a6b359efecb1856d20cd0343db8 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 27 May 2010 15:53:05 -0400 Subject: oops, forgot this guy --- src/fabric_doc.erl | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/fabric_doc.erl diff --git a/src/fabric_doc.erl b/src/fabric_doc.erl new file mode 100644 index 00000000..463a3a9b --- /dev/null +++ b/src/fabric_doc.erl @@ -0,0 +1,76 @@ +-module(fabric_doc). +-export([open_doc/3, open_doc_revs/4, get_missing_revs/2, update_docs/3]). + +-include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). + +open_doc(DbName, DocId, Opts) -> + Shards = partitions:for_key(DbName, DocId), + Workers = fabric_util:submit_jobs(Shards, open_doc, [DocId, [deleted|Opts]]), + Acc0 = {length(Workers), couch_util:get_value(r, Opts, 1), []}, + ?LOG_INFO("Workers ~p Acc0 ~p", [Workers, Acc0]), + SuppressDeletedDoc = not lists:member(deleted, Opts), + case fabric_util:recv(Workers, #shard.ref, fun handle_open_doc/3, Acc0) of + {ok, #doc{deleted=true}} when SuppressDeletedDoc -> + {not_found, deleted}; + Else -> + Else + end. + +open_doc_revs(DbName, DocId, Revs, Options) -> + ok. + +update_docs(DbName, Docs, Options) -> + ok. + +get_missing_revs(DbName, IdsRevs) -> + ok. + +handle_open_doc(_Worker, {rexi_DOWN, _, _, _}, {WaitingCount, R, Replies}) -> + if WaitingCount =:= 1 -> + repair_read_quorum_failure(Replies); + true -> + {ok, {WaitingCount-1, R, Replies}} + end; +handle_open_doc(_Worker, {rexi_EXIT, Reason}, {WaitingCount, R, Replies}) -> + ?LOG_ERROR("open_doc rexi_EXIT ~p", [Reason]), + if WaitingCount =:= 1 -> + repair_read_quorum_failure(Replies); + true -> + {ok, {WaitingCount-1, R, Replies}} + end; +handle_open_doc(_Worker, Reply, {WaitingCount, R, Replies}) -> + ?LOG_INFO("got ~p when ~p ~p ~p", [Reply, WaitingCount, R, Replies]), + case merge_replies(make_key(Reply), Reply, Replies) of + {_, KeyCount} when KeyCount =:= R -> + {stop, Reply}; + {NewReplies, KeyCount} when KeyCount < R -> + if WaitingCount =:= 1 -> + % last message arrived, but still no quorum + repair_read_quorum_failure(NewReplies); + true -> + {ok, {WaitingCount-1, R, NewReplies}} + end + end. + +merge_replies(Key, Reply, Replies) -> + case lists:keyfind(Key, 1, Replies) of + false -> + {[{Key, Reply, 1} | Replies], 1}; + {Key, _, N} -> + {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} + end. + +make_key({ok, #doc{id=Id, revs=Revs}}) -> + {Id, Revs}; +make_key({not_found, missing}) -> + {not_found, missing}. + +repair_read_quorum_failure(Replies) -> + case [Doc || {ok, Doc} <- Replies] of + [] -> + {stop, {not_found, missing}}; + [Doc|Rest] -> + % TODO merge docs to find the winner as determined by replication + {stop, {ok, Doc}} + end. -- cgit v1.2.3 From 591eb31af0ad55e528358608f352474283b47430 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 27 May 2010 16:02:40 -0400 Subject: work on get_db_info, conflict resovled for fabric_rpc --- src/fabric_create.erl | 4 +-- src/fabric_info.erl | 67 ++++++++++++++++++++++++++++----------------------- src/fabric_rpc.erl | 5 ++++ src/fabric_util.erl | 2 ++ 4 files changed, 46 insertions(+), 32 deletions(-) diff --git a/src/fabric_create.erl b/src/fabric_create.erl index 17c85b6a..79bf1040 100644 --- a/src/fabric_create.erl +++ b/src/fabric_create.erl @@ -19,8 +19,8 @@ create_db(DbName, Options) -> Fullmap = partitions:fullmap(DbName, Options), {ok, FullNodes} = mem3:fullnodes(), RefPartMap = send_create_calls(DbName, Options, Fullmap), - Acc0 = {false, length(RefPartMap), - lists:usort([ {Beg, false} || {_,#shard{range=[Beg,_]}} <- RefPartMap])}, + Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || + {_,#shard{range=[Beg,_]}} <- RefPartMap])}, case fabric_util:receive_loop( RefPartMap, 1, fun handle_create_msg/3, Acc0, 5000, infinity) of {ok, _Results} -> diff --git a/src/fabric_info.erl b/src/fabric_info.erl index 662c06b6..cb32eac0 100644 --- a/src/fabric_info.erl +++ b/src/fabric_info.erl @@ -25,13 +25,19 @@ all_databases(Customer) -> {ok, Dbs}. %% @doc get database information tuple -get_db_info(DbName, _Customer) -> +get_db_info(DbName, Customer) -> + Name = cloudant_db_name(Customer, DbName), Parts = partitions:all_parts(DbName), RefPartMap = send_info_calls(DbName, Parts), - {ok, Results} = fabric_rpc:receive_loop(RefPartMap, 5000, fun info_loop/3), - InfoList = Results, - % process_infos(ShardInfos, [{db_name, Name}]); - {ok, InfoList}. + Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, nil} || + {_,#shard{range=[Beg,_]}} <- RefPartMap])}, + case fabric_util:receive_loop( + RefPartMap, 1, fun handle_info_msg/3, Acc0, 5000, infinity) of + {ok, ShardInfos} -> + {ok, process_infos(ShardInfos, [{db_name, Name}])}; + Error -> Error + end. + %% ===================== %% internal @@ -46,37 +52,38 @@ new_acc(DbName, Acc) -> send_info_calls(DbName, Parts) -> lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> ShardName = showroom_utils:shard_name(Beg, DbName), - Ref = rexi:cast(Node, {rexi_rpc, get_db_info, ShardName}), + Ref = rexi:cast(Node, {fabric_rpc, get_db_info, ShardName}), {Ref, Part} end, Parts). -%% @doc create_db receive loop -%% Acc is either an accumulation of responses, or if we've received all -%% responses, it's {ok, Responses} --spec info_loop([ref_part_map()], tref(), beg_acc()) -> - beg_acc() | {ok, beg_acc()}. -info_loop(_,_,{ok, Acc}) -> {ok, Acc}; -info_loop(RefPartMap, TimeoutRef, AccIn) -> - receive - {Ref, {ok, Info}} when is_reference(Ref) -> - %AccOut = check_all_parts(Ref, RefPartMap, AccIn, ok), - %info_loop(RefPartMap, TimeoutRef, AccOut); - ok; - {Ref, Reply} when is_reference(Ref) -> - %AccOut = check_all_parts(Ref, RefPartMap, AccIn, Reply), - %info_loop(RefPartMap, TimeoutRef, AccOut); - ok; - {timeout, TimeoutRef} -> - {error, timeout} - end. - +handle_info_msg(_, _, {true, _, Infos0}) -> + {stop, Infos0}; +handle_info_msg(_, _, {false, 1, Infos0}) -> + MissingShards = lists:keyfind(nil, 2, Infos0), + ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), + {error, get_db_info}; +handle_info_msg({_,#shard{range=[Beg,_]}}, {ok, Info}, {false, N, Infos0}) -> + case couch_util:get_value(Beg, Infos0) of + nil -> + Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), + case is_complete(Infos) of + true -> {ok, {true, N-1, Infos}}; + false -> {ok, {false, N-1, Infos}} + end; + _ -> + {ok, {false, N-1, Infos0}} + end; +handle_info_msg(_, _Other, {Complete, N, Infos0}) -> + {ok, {Complete, N-1, Infos0}}. + + +is_complete(List) -> + not lists:any(fun({_,nil}) -> true end, List). cloudant_db_name(Customer, FullName) -> case Customer of - "" -> - FullName; - Name -> - re:replace(FullName, [Name,"/"], "", [{return, binary}]) + "" -> FullName; + Name -> re:replace(FullName, [Name,"/"], "", [{return, binary}]) end. %% Loop through Tasks on the flattened Infos and get the aggregated result diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 49809145..54f3b338 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -2,13 +2,18 @@ -export([open_doc/3, open_doc/4, get_db_info/1]). +-include_lib("eunit/include/eunit.hrl"). open_doc(DbName, DocId, Options) -> with_db(DbName, {couch_db, open_doc, [DocId, Options]}). +%% rpc endpoints +%% call to with_db will supply your M:F with a #db{} and then remaining args + open_doc(DbName, DocId, Revs, Options) -> with_db(DbName, {couch_api, open_doc, [DocId, Revs, Options]}). get_db_info(DbName) -> + ?debugHere, with_db(DbName, {couch_db, get_db_info, []}). %% diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 4b2f611a..347670a4 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -3,6 +3,7 @@ -export([submit_jobs/3, recv/4, receive_loop/4, receive_loop/6]). -include("../../dynomite/include/membership.hrl"). +-include_lib("eunit/include/eunit.hrl"). submit_jobs(Shards, EndPoint, ExtraArgs) -> lists:map(fun(#shard{node=Node, name=ShardName} = Shard) -> @@ -55,6 +56,7 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> {ok, Acc0}; RefPart -> % call the Fun that understands the message + ?debugFmt("~nAcc0: ~p~n", [Acc0]), Fun(RefPart, Msg, Acc0) end; {rexi_DOWN, _RexiMonPid, ServerPid, Reason} = Msg -> -- cgit v1.2.3 From faedecbc0393d87523fbc19b9edc17504c504782 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 27 May 2010 16:43:32 -0400 Subject: get_db_info working, also reorg'd fabric to doc and db module structure --- ebin/fabric.app | 9 +-- src/fabric.erl | 16 ++-- src/fabric_create.erl | 72 ----------------- src/fabric_db.erl | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/fabric_delete.erl | 65 --------------- src/fabric_info.erl | 122 ---------------------------- src/fabric_open.erl | 47 ----------- src/fabric_rpc.erl | 1 - 8 files changed, 229 insertions(+), 321 deletions(-) delete mode 100644 src/fabric_create.erl create mode 100644 src/fabric_db.erl delete mode 100644 src/fabric_delete.erl delete mode 100644 src/fabric_info.erl delete mode 100644 src/fabric_open.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index cd81b383..60f1f634 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -6,13 +6,10 @@ {modules, [ fabric, fabric_api, - fabric_create, - fabric_delete, - fabric_info, - fabric_open, + fabric_db, + fabric_doc, fabric_rpc, - fabric_util, - fabric_doc + fabric_util ]}, {registered, []}, {included_applications, []}, diff --git a/src/fabric.erl b/src/fabric.erl index 15269bbb..e75e31eb 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -3,23 +3,23 @@ -export([all_databases/1, create_db/2, delete_db/2, open_doc/3, open_doc/4, get_db_info/2]). -%% maybe this goes away, and these are called directly in their own modules in -%% fabric_api ?? +% db operations all_databases(Customer) -> - fabric_info:all_databases(Customer). + fabric_db:all_databases(Customer). + +get_db_info(DbName, Customer) -> + fabric_db:get_db_info(DbName, Customer). create_db(DbName, Options) -> - fabric_create:create_db(DbName, Options). + fabric_db:create_db(DbName, Options). delete_db(DbName, Options) -> - fabric_delete:delete_db(DbName, Options). + fabric_db:delete_db(DbName, Options). +% doc operations open_doc(Db, DocId, Options) -> fabric_doc:open_doc(Db, DocId, Options). open_doc(Db, DocId, Revs, Options) -> fabric_open:open_doc(Db, DocId, Revs, Options). - -get_db_info(DbName, Customer) -> - fabric_info:get_db_info(DbName, Customer). diff --git a/src/fabric_create.erl b/src/fabric_create.erl deleted file mode 100644 index 79bf1040..00000000 --- a/src/fabric_create.erl +++ /dev/null @@ -1,72 +0,0 @@ --module(fabric_create). --author('Brad Anderson '). - --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - -%% api --export([create_db/2]). - - -%% ===================== -%% api -%% ===================== - -%% @doc Create a new database, and all its partition files across the cluster -%% Options is proplist with user_ctx, n, q --spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. -create_db(DbName, Options) -> - Fullmap = partitions:fullmap(DbName, Options), - {ok, FullNodes} = mem3:fullnodes(), - RefPartMap = send_create_calls(DbName, Options, Fullmap), - Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || - {_,#shard{range=[Beg,_]}} <- RefPartMap])}, - case fabric_util:receive_loop( - RefPartMap, 1, fun handle_create_msg/3, Acc0, 5000, infinity) of - {ok, _Results} -> - partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), - ok; - Error -> Error - end. - - -%% ===================== -%% internal -%% ===================== - -%% @doc create the partitions on all appropriate nodes (rexi calls) --spec send_create_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. -send_create_calls(DbName, Options, Fullmap) -> - lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> - ShardName = showroom_utils:shard_name(Beg, DbName), - Ref = rexi:async_server_call({couch_server, Node}, - {create, ShardName, Options}), - {Ref, Part} - end, Fullmap). - -handle_create_msg(_, file_exists, _) -> - {error, file_exists}; -handle_create_msg(_, {rexi_EXIT, _Reason}, {Complete, N, Parts}) -> - {ok, {Complete, N-1, Parts}}; -handle_create_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> - if - Complete -> {stop, ok}; - true -> {error, create_db_fubar} - end; -handle_create_msg(_, _, {true, 1, _Acc}) -> - {stop, ok}; -handle_create_msg({_, #shard{range=[Beg,_]}}, {ok, _}, {false, 1, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - case is_complete(PartResults) of - true -> {stop, ok}; - false -> {error, create_db_fubar} - end; -handle_create_msg(_RefPart, {ok, _}, {true, N, Parts}) -> - {ok, {true, N-1, Parts}}; -handle_create_msg({_Ref, #shard{range=[Beg,_]}}, {ok, _}, {false, Rem, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - {ok, {is_complete(PartResults), Rem-1, PartResults}}. - - -is_complete(List) -> - lists:all(fun({_,Bool}) -> Bool end, List). diff --git a/src/fabric_db.erl b/src/fabric_db.erl new file mode 100644 index 00000000..95648e01 --- /dev/null +++ b/src/fabric_db.erl @@ -0,0 +1,218 @@ +-module(fabric_db). +-author('Brad Anderson '). +-author('Adam Kocoloski '). + +-export([all_databases/1, get_db_info/2, create_db/2, delete_db/2]). + +-include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). + +%% @doc gets all databases in the cluster. +-spec all_databases(binary() | []) -> [binary()]. +all_databases([]) -> + Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> + new_acc(DbName, AccIn) + end, [], partitions), + {ok, Dbs}; +all_databases(Customer) -> + ?debugFmt("~nCustomer: ~p~n", [Customer]), + Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> + DbNameStr = ?b2l(DbName), + case string:str(DbNameStr, Customer) of + 1 -> + new_acc(DbNameStr, AccIn); + _ -> AccIn + end + end, [], dbs_cache), + {ok, Dbs}. + +%% @doc get database information tuple +get_db_info(DbName, Customer) -> + Name = cloudant_db_name(Customer, DbName), + Shards = partitions:all_parts(DbName), + Workers = fabric_util:submit_jobs(Shards, get_db_info, []), + Acc0 = {false, length(Workers), lists:usort([ {Beg, nil} || + #shard{range=[Beg,_]} <- Workers])}, + case fabric_util:recv(Workers, #shard.ref, fun handle_info_msg/3, Acc0) of + {ok, ShardInfos} -> + {ok, process_infos(ShardInfos, [{db_name, Name}])}; + Error -> Error + end. + +%% @doc Create a new database, and all its partition files across the cluster +%% Options is proplist with user_ctx, n, q +-spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. +create_db(DbName, Options) -> + Fullmap = partitions:fullmap(DbName, Options), + {ok, FullNodes} = mem3:fullnodes(), + RefPartMap = send_create_calls(DbName, Options, Fullmap), + Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || + {_,#shard{range=[Beg,_]}} <- RefPartMap])}, + case fabric_util:receive_loop( + RefPartMap, 1, fun handle_create_msg/3, Acc0, 5000, infinity) of + {ok, _Results} -> + partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), + ok; + Error -> Error + end. + +%% @doc Delete a database, and all its partition files across the cluster +%% Options is proplist with user_ctx, n, q +-spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. +delete_db(DbName, Options) -> + Parts = partitions:all_parts(DbName), + RefPartMap = send_delete_calls(DbName, Options, Parts), + Acc0 = {true, length(RefPartMap)}, + case fabric_util:receive_loop( + RefPartMap, 1, fun handle_delete_msg/3, Acc0, 5000, infinity) of + {ok, _Results} -> + delete_fullmap(DbName), + ok; + Error -> Error + end. + +%% ===================== +%% internal +%% ===================== + +new_acc(DbName, Acc) -> + case lists:member(DbName, Acc) of + true -> Acc; + _ ->[DbName | Acc] + end. + +handle_info_msg(_, _, {true, _, Infos0}) -> + {stop, Infos0}; +handle_info_msg(_, _, {false, 1, Infos0}) -> + MissingShards = lists:reverse(lists:foldl(fun + ({S,nil}, Acc) -> [S|Acc]; + (_, Acc) -> Acc + end, [], Infos0)), + ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), + {error, get_db_info}; +handle_info_msg(#shard{range=[Beg,_]}, {ok, Info}, {false, N, Infos0}) -> + case couch_util:get_value(Beg, Infos0) of + nil -> + Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), + case is_complete(info, Infos) of + true -> {ok, {true, N-1, Infos}}; + false -> {ok, {false, N-1, Infos}} + end; + _ -> + {ok, {false, N-1, Infos0}} + end; +handle_info_msg(_, _Other, {Complete, N, Infos0}) -> + {ok, {Complete, N-1, Infos0}}. + +is_complete(info, List) -> + not lists:any(fun({_,Info}) -> Info =:= nil end, List); +is_complete(create, List) -> + lists:all(fun({_,Bool}) -> Bool end, List). + +cloudant_db_name(Customer, FullName) -> + case Customer of + "" -> FullName; + Name -> re:replace(FullName, [Name,"/"], "", [{return, binary}]) + end. + +%% Loop through Tasks on the flattened Infos and get the aggregated result +process_infos(Infos, Initial) -> + Tasks = [ + {doc_count, fun sum/2, 0}, + {doc_del_count, fun sum/2, 0}, + {update_seq, fun max/2, 1}, + {purge_seq, fun sum/2, 0}, + {compact_running, fun bool/2, 0}, + {disk_size, fun sum/2, 0}, + {instance_start_time, fun(_, _) -> <<"0">> end, 0}, + {disk_format_version, fun max/2, 0}], + + Infos1 = lists:flatten(Infos), + + Result = lists:map(fun({Type, Fun, Default}) -> + {Type, process_info(Type, Fun, Default, Infos1)} + end, Tasks), + lists:flatten([Initial, Result]). + + process_info(Type, Fun, Default, List) -> + lists:foldl(fun(V, AccIn) -> Fun(V, AccIn) end, Default, + proplists:get_all_values(Type, List)). + +sum(New, Existing) -> + New + Existing. + +bool(New, Existing) -> + New andalso Existing. + +max(New, Existing) -> + case New > Existing of + true -> New; + false -> Existing + end. + + +%% @doc create the partitions on all appropriate nodes (rexi calls) +-spec send_create_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. +send_create_calls(DbName, Options, Fullmap) -> + lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> + ShardName = showroom_utils:shard_name(Beg, DbName), + Ref = rexi:async_server_call({couch_server, Node}, + {create, ShardName, Options}), + {Ref, Part} + end, Fullmap). + +handle_create_msg(_, file_exists, _) -> + {error, file_exists}; +handle_create_msg(_, {rexi_EXIT, _Reason}, {Complete, N, Parts}) -> + {ok, {Complete, N-1, Parts}}; +handle_create_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> + if + Complete -> {stop, ok}; + true -> {error, create_db_fubar} + end; +handle_create_msg(_, _, {true, 1, _Acc}) -> + {stop, ok}; +handle_create_msg({_, #shard{range=[Beg,_]}}, {ok, _}, {false, 1, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + case is_complete(create, PartResults) of + true -> {stop, ok}; + false -> {error, create_db_fubar} + end; +handle_create_msg(_RefPart, {ok, _}, {true, N, Parts}) -> + {ok, {true, N-1, Parts}}; +handle_create_msg({_Ref, #shard{range=[Beg,_]}}, {ok, _}, {false, Rem, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + {ok, {is_complete(create, PartResults), Rem-1, PartResults}}. + + +%% @doc delete the partitions on all appropriate nodes (rexi calls) +-spec send_delete_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. +send_delete_calls(DbName, Options, Parts) -> + lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> + ShardName = showroom_utils:shard_name(Beg, DbName), + Ref = rexi:async_server_call({couch_server, Node}, + {delete, ShardName, Options}), + {Ref, Part} + end, Parts). + +handle_delete_msg(_, not_found, {NotFound, N}) -> + {ok, {NotFound, N-1}}; +handle_delete_msg(_, {rexi_EXIT, _Reason}, {NotFound, N}) -> + {ok, {NotFound, N-1}}; +handle_delete_msg(_, {rexi_DOWN, _, _, _}, _Acc) -> + {error, delete_db_fubar}; +handle_delete_msg(_, _, {NotFound, 1}) -> + if + NotFound -> {stop, not_found}; + true -> {stop, ok} + end; +handle_delete_msg(_, ok, {_NotFound, N}) -> + {ok, {false, N-1}}. + +delete_fullmap(DbName) -> + case couch_db:open(<<"dbs">>, []) of + {ok, Db} -> + {ok, Doc} = couch_api:open_doc(Db, DbName, nil, []), + couch_api:update_doc(Db, Doc#doc{deleted=true}); + Error -> Error + end. diff --git a/src/fabric_delete.erl b/src/fabric_delete.erl deleted file mode 100644 index 9a65f72b..00000000 --- a/src/fabric_delete.erl +++ /dev/null @@ -1,65 +0,0 @@ --module(fabric_delete). --author('Brad Anderson '). - --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - -%% api --export([delete_db/2]). - - -%% ===================== -%% api -%% ===================== - -%% @doc Delete a database, and all its partition files across the cluster -%% Options is proplist with user_ctx, n, q --spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. -delete_db(DbName, Options) -> - Parts = partitions:all_parts(DbName), - RefPartMap = send_calls(DbName, Options, Parts), - Acc0 = {true, length(RefPartMap)}, - case fabric_util:receive_loop( - RefPartMap, 1, fun handle_delete_msg/3, Acc0, 5000, infinity) of - {ok, _Results} -> - delete_fullmap(DbName), - ok; - Error -> Error - end. - - -%% ===================== -%% internal -%% ===================== - -%% @doc delete the partitions on all appropriate nodes (rexi calls) --spec send_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. -send_calls(DbName, Options, Parts) -> - lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> - ShardName = showroom_utils:shard_name(Beg, DbName), - Ref = rexi:async_server_call({couch_server, Node}, - {delete, ShardName, Options}), - {Ref, Part} - end, Parts). - -handle_delete_msg(_, not_found, {NotFound, N}) -> - {ok, {NotFound, N-1}}; -handle_delete_msg(_, {rexi_EXIT, _Reason}, {NotFound, N}) -> - {ok, {NotFound, N-1}}; -handle_delete_msg(_, {rexi_DOWN, _, _, _}, _Acc) -> - {error, delete_db_fubar}; -handle_delete_msg(_, _, {NotFound, 1}) -> - if - NotFound -> {stop, not_found}; - true -> {stop, ok} - end; -handle_delete_msg(_, ok, {_NotFound, N}) -> - {ok, {false, N-1}}. - -delete_fullmap(DbName) -> - case couch_db:open(<<"dbs">>, []) of - {ok, Db} -> - {ok, Doc} = couch_api:open_doc(Db, DbName, nil, []), - couch_api:update_doc(Db, Doc#doc{deleted=true}); - Error -> Error - end. diff --git a/src/fabric_info.erl b/src/fabric_info.erl deleted file mode 100644 index cb32eac0..00000000 --- a/src/fabric_info.erl +++ /dev/null @@ -1,122 +0,0 @@ --module(fabric_info). - --export([all_databases/1, get_db_info/2]). - --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - -%% @doc gets all databases in the cluster. --spec all_databases(binary() | []) -> [binary()]. -all_databases([]) -> - Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> - new_acc(DbName, AccIn) - end, [], partitions), - {ok, Dbs}; -all_databases(Customer) -> - ?debugFmt("~nCustomer: ~p~n", [Customer]), - Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> - DbNameStr = ?b2l(DbName), - case string:str(DbNameStr, Customer) of - 1 -> - new_acc(DbNameStr, AccIn); - _ -> AccIn - end - end, [], dbs_cache), - {ok, Dbs}. - -%% @doc get database information tuple -get_db_info(DbName, Customer) -> - Name = cloudant_db_name(Customer, DbName), - Parts = partitions:all_parts(DbName), - RefPartMap = send_info_calls(DbName, Parts), - Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, nil} || - {_,#shard{range=[Beg,_]}} <- RefPartMap])}, - case fabric_util:receive_loop( - RefPartMap, 1, fun handle_info_msg/3, Acc0, 5000, infinity) of - {ok, ShardInfos} -> - {ok, process_infos(ShardInfos, [{db_name, Name}])}; - Error -> Error - end. - - -%% ===================== -%% internal -%% ===================== - -new_acc(DbName, Acc) -> - case lists:member(DbName, Acc) of - true -> Acc; - _ ->[DbName | Acc] - end. - -send_info_calls(DbName, Parts) -> - lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> - ShardName = showroom_utils:shard_name(Beg, DbName), - Ref = rexi:cast(Node, {fabric_rpc, get_db_info, ShardName}), - {Ref, Part} - end, Parts). - -handle_info_msg(_, _, {true, _, Infos0}) -> - {stop, Infos0}; -handle_info_msg(_, _, {false, 1, Infos0}) -> - MissingShards = lists:keyfind(nil, 2, Infos0), - ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), - {error, get_db_info}; -handle_info_msg({_,#shard{range=[Beg,_]}}, {ok, Info}, {false, N, Infos0}) -> - case couch_util:get_value(Beg, Infos0) of - nil -> - Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), - case is_complete(Infos) of - true -> {ok, {true, N-1, Infos}}; - false -> {ok, {false, N-1, Infos}} - end; - _ -> - {ok, {false, N-1, Infos0}} - end; -handle_info_msg(_, _Other, {Complete, N, Infos0}) -> - {ok, {Complete, N-1, Infos0}}. - - -is_complete(List) -> - not lists:any(fun({_,nil}) -> true end, List). - -cloudant_db_name(Customer, FullName) -> - case Customer of - "" -> FullName; - Name -> re:replace(FullName, [Name,"/"], "", [{return, binary}]) - end. - -%% Loop through Tasks on the flattened Infos and get the aggregated result -process_infos(Infos, Initial) -> - Tasks = [ - {doc_count, fun sum/2, 0}, - {doc_del_count, fun sum/2, 0}, - {update_seq, fun max/2, 1}, - {purge_seq, fun sum/2, 0}, - {compact_running, fun bool/2, 0}, - {disk_size, fun sum/2, 0}, - {instance_start_time, fun(_, _) -> <<"0">> end, 0}, - {disk_format_version, fun max/2, 0}], - - Infos1 = lists:flatten(Infos), - - Result = lists:map(fun({Type, Fun, Default}) -> - {Type, process_info(Type, Fun, Default, Infos1)} - end, Tasks), - lists:flatten([Initial, Result]). - - process_info(Type, Fun, Default, List) -> - lists:foldl(fun(V, AccIn) -> Fun(V, AccIn) end, Default, - couch_util:get_all_values(Type, List)). - -sum(New, Existing) -> - New + Existing. - -bool(New, Existing) -> - New andalso Existing. - -max(New, Existing) -> - case New > Existing of - true -> New; - false -> Existing - end. diff --git a/src/fabric_open.erl b/src/fabric_open.erl deleted file mode 100644 index cab10c5d..00000000 --- a/src/fabric_open.erl +++ /dev/null @@ -1,47 +0,0 @@ --module(fabric_open). - --export([open_doc/4]). - --include("../../couch/src/couch_db.hrl"). - - -% open_db(<<"S", ShardFileName/binary>> = Name, Options) -> -% case couch_db:open(ShardFileName, Options) of -% {ok, Db} -> -% {ok, Db#db{name = Name}}; -% {not_found, no_db_file} -> -% {not_found, no_db_file} -% end; -% -% open_db(DbName, Options) -> -% Part = case lists:keyfind(node(), 1, membership2:all_nodes_parts(false)) of -% {_, P} -> P; -% _ -> throw({node_has_no_parts, node()}) -% end, -% ShardName = partitions:shard_name(Part, DbName), -% open_db(<<"S", ShardName/binary>>, Options). - - -open_doc(DbName, DocId, _Revs, _Options) -> - NPs = partitions:key_nodes_parts(DbName, DocId), - ?debugFmt("~nNPs: ~p~n", [NPs]), - {ok, #doc{}}. - - - - - -% open_doc(Db, DocId, Revs, Options) -> -% {R,N} = get_quorum_constants(r, Options), -% case cluster_ops:key_lookup(DocId, {dynomite_couch_api, get, -% [Db, DocId, Revs, Options]}, r, R, N) of -% {ok, [Doc|_Rest]} -> -% {ok, Doc}; -% {ok, {not_found, Reason}, _Reasons} -> -% {not_found, Reason}; -% [{error, Error}, {good, Good}, {bad, Bad}] -> -% showroom_quorum_utils:handle_error(Error, Good, Bad); -% Other -> -% ?LOG_DEBUG("~nopen_doc Other: ~p~n", [Other]), -% throw(Other) -% end. diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 54f3b338..a0c0a568 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -13,7 +13,6 @@ open_doc(DbName, DocId, Revs, Options) -> with_db(DbName, {couch_api, open_doc, [DocId, Revs, Options]}). get_db_info(DbName) -> - ?debugHere, with_db(DbName, {couch_db, get_db_info, []}). %% -- cgit v1.2.3 From 6c7e3665cd941083dedb8ead5e9314f3c531ff89 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Thu, 27 May 2010 17:09:19 -0400 Subject: remove fabric_api, clean up other things to get futon rendering db list --- ebin/fabric.app | 1 - src/fabric.erl | 31 +++++++++++++- src/fabric_api.erl | 118 ---------------------------------------------------- src/fabric_doc.erl | 2 + src/fabric_util.erl | 2 +- 5 files changed, 32 insertions(+), 122 deletions(-) delete mode 100644 src/fabric_api.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index 60f1f634..e08f560b 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -5,7 +5,6 @@ {vsn, "0.1.0"}, {modules, [ fabric, - fabric_api, fabric_db, fabric_doc, fabric_rpc, diff --git a/src/fabric.erl b/src/fabric.erl index e75e31eb..0957387e 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,10 +1,18 @@ -module(fabric). -export([all_databases/1, create_db/2, delete_db/2, open_doc/3, open_doc/4, - get_db_info/2]). + get_db_info/2, db_path/2]). + +-include("../../couch/src/couch_db.hrl"). % db operations +-spec db_path(bstring(), bstring()) -> bstring(). +db_path(RawUri, Customer) -> + CustomerUri = generate_customer_path(RawUri, Customer), + {Path, _, _} = mochiweb_util:urlsplit_path(CustomerUri), + Path. + all_databases(Customer) -> fabric_db:all_databases(Customer). @@ -22,4 +30,23 @@ open_doc(Db, DocId, Options) -> fabric_doc:open_doc(Db, DocId, Options). open_doc(Db, DocId, Revs, Options) -> - fabric_open:open_doc(Db, DocId, Revs, Options). + fabric_doc:open_doc(Db, DocId, Revs, Options). + + + +%% +%% internal +%% +generate_customer_path("/", _Customer) -> + ""; +generate_customer_path("/favicon.ico", _Customer) -> + "favicon.ico"; +generate_customer_path([$/,$_|Rest], _Customer) -> + lists:flatten([$_|Rest]); +generate_customer_path([$/|RawPath], Customer) -> + case Customer of + "" -> + RawPath; + Else -> + lists:flatten([Else, "%2F", RawPath]) + end. diff --git a/src/fabric_api.erl b/src/fabric_api.erl deleted file mode 100644 index 53dc96e3..00000000 --- a/src/fabric_api.erl +++ /dev/null @@ -1,118 +0,0 @@ -%% This is a raw Erlang API for CouchDB in a Cloudant cluster -%% It makes use of clustering facilities - --module(fabric_api). --author('adam@cloudant.com'). --author('brad@cloudant.com'). - --include("../../couch/src/couch_db.hrl"). - --compile(export_all). - --type response() :: any(). - -%% dialyzer doesn't have recursive types, so this is necessarily wrong --type ejson_value() :: true | false | number() | bstring() | list(). --type ejson() :: {[{bstring(), ejson_value()}]}. - -%% Database - --spec db_path(bstring(), bstring()) -> bstring(). -db_path(RawUri, Customer) -> - showroom_db:db_path(RawUri, Customer). - --spec all_databases(string()) -> {ok, [bstring()]}. -all_databases(Customer) -> - fabric:all_databases(Customer). - --spec create_db(bstring(), [any()]) -> {ok, #db{}} | {error, any()}. -create_db(DbName, Options) -> - fabric:create_db(DbName, Options). - --spec delete_db(bstring(), [any()]) -> ok | not_found | {error, atom()}. -delete_db(DbName, Options) -> - fabric:delete_db(DbName, Options). - --spec open_db(bstring(), [any()]) -> {ok, #db{}} | {error, any()}. -open_db(DbName, Options) -> - fabric:open_db(DbName, Options). - --spec close_db(#db{}) -> ok. -close_db(Db) -> - showroom_db:close_db(Db). - --spec get_db_info(#db{}, bstring()) -> {ok, [{atom(), any()}]}. -get_db_info(Db, Customer) -> - fabric:get_db_info(Db, Customer). - --spec replicate_db(ejson(), #user_ctx{}) -> {ok, ejson()}. -replicate_db(PostBody, UserCtx) -> - showroom_rep:replicate(PostBody, UserCtx). - --spec ensure_full_commit(#db{}) -> {ok, InstanceStartTime::bstring()}. -ensure_full_commit(Db) -> - showroom_db:ensure_full_commit(Db), - {ok, <<"0">>}. - - -%% Document - -att_receiver(Req, Length) -> - showroom_att:receiver(Req, Length). - --spec get_missing_revs(#db{}, [{docid(), [revision()]}]) -> - {ok, [{docid(), [revision()]}]}. -get_missing_revs(Db, IdsRevs) -> - showroom_doc:get_missing_revs(Db, IdsRevs). - -open_doc(DbName, DocId) -> - open_doc(DbName, DocId, nil, []). - --spec open_doc(bstring(), docid(), [any()]) -> - {ok, #doc{}} | {not_found, deleted | missing}. -open_doc(DbName, DocId, Options) -> - open_doc(DbName, DocId, nil, Options). - -open_doc(DbName, DocId, Revs, Options) -> - fabric:open_doc(DbName, DocId, Revs, Options). - --spec open_doc_revs(bstring(), docid(), [revision()], [any()]) -> - {ok, [{ok, #doc{}} - | {{not_found, deleted | missing}, revision()}]}. -open_doc_revs(DbName, DocId, Revs, Options) -> - open_doc(DbName, DocId, Revs, Options). - -update_doc(Db, Doc) -> - update_doc(Db, Doc, []). - --spec update_doc(#db{}, #doc{}, [any()]) -> {ok, revision()}. -update_doc(Db, Doc, Options) -> - showroom_doc:update_doc(Db, Doc, Options). - -update_docs(Db, Docs, Options) -> - update_docs(Db, Docs, Options, interactive_edit). - --spec update_docs(#db{}, [#doc{}], [any()], interactive_edit | - replicated_changes) -> {ok, [{ok, revision()}]}. -update_docs(Db, Docs, Options, Type) -> - showroom_doc:update_docs(Db, Docs, Options, Type). - - -%% View - --spec all_docs_view(response(), #db{}, nil | list(), #view_query_args{}) -> - {ok, any()}. -all_docs_view(Resp, Db, Keys, QueryArgs) -> - showroom_view:all_docs(Resp, Db, Keys, QueryArgs). - --spec design_view(response(), #db{}, bstring(), bstring(), nil | list(), - #view_query_args{}) -> any(). -design_view(Resp, Db, Id, Name, Keys, QueryArgs) -> - showroom_view:design(Resp, Db, Id, Name, Keys, QueryArgs). - -list_view(Req, Db, DesignId, ViewName, Keys, QueryArgs, QueryServer) -> - showroom_view:list(Req, Db, DesignId, ViewName, Keys, QueryArgs, QueryServer). - --spec get_view_group_info(#db{}, bstring()) -> {ok, [{atom(), any()}]}. -get_view_group_info(Db, DesignId) -> - showroom_view:group_info(Db, DesignId). diff --git a/src/fabric_doc.erl b/src/fabric_doc.erl index 463a3a9b..407a9187 100644 --- a/src/fabric_doc.erl +++ b/src/fabric_doc.erl @@ -13,6 +13,8 @@ open_doc(DbName, DocId, Opts) -> case fabric_util:recv(Workers, #shard.ref, fun handle_open_doc/3, Acc0) of {ok, #doc{deleted=true}} when SuppressDeletedDoc -> {not_found, deleted}; + {ok, {not_found, missing}} -> + {not_found, missing}; Else -> Else end. diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 347670a4..38f0b9f3 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -56,7 +56,7 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> {ok, Acc0}; RefPart -> % call the Fun that understands the message - ?debugFmt("~nAcc0: ~p~n", [Acc0]), + %?debugFmt("~nAcc0: ~p~n", [Acc0]), Fun(RefPart, Msg, Acc0) end; {rexi_DOWN, _RexiMonPid, ServerPid, Reason} = Msg -> -- cgit v1.2.3 From a60895fedef53626b622f04af4d20a5792cfbd7d Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 10:17:30 -0400 Subject: lots of work, expecially update_docs --- src/fabric_doc.erl | 143 +++++++++++++++++++++++++++++++++++++++++----------- src/fabric_rpc.erl | 8 ++- src/fabric_util.erl | 2 + 3 files changed, 122 insertions(+), 31 deletions(-) diff --git a/src/fabric_doc.erl b/src/fabric_doc.erl index 407a9187..c3cc0f47 100644 --- a/src/fabric_doc.erl +++ b/src/fabric_doc.erl @@ -1,15 +1,14 @@ -module(fabric_doc). --export([open_doc/3, open_doc_revs/4, get_missing_revs/2, update_docs/3]). +-export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). -include("../../couch/src/couch_db.hrl"). -include("../../dynomite/include/membership.hrl"). -open_doc(DbName, DocId, Opts) -> - Shards = partitions:for_key(DbName, DocId), - Workers = fabric_util:submit_jobs(Shards, open_doc, [DocId, [deleted|Opts]]), - Acc0 = {length(Workers), couch_util:get_value(r, Opts, 1), []}, - ?LOG_INFO("Workers ~p Acc0 ~p", [Workers, Acc0]), - SuppressDeletedDoc = not lists:member(deleted, Opts), +open_doc(DbName, Id, Options) -> + Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_doc, + [Id, Options]), + SuppressDeletedDoc = not lists:member(deleted, Options), + Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, case fabric_util:recv(Workers, #shard.ref, fun handle_open_doc/3, Acc0) of {ok, #doc{deleted=true}} when SuppressDeletedDoc -> {not_found, deleted}; @@ -19,31 +18,36 @@ open_doc(DbName, DocId, Opts) -> Else end. -open_doc_revs(DbName, DocId, Revs, Options) -> - ok. +open_revs(DbName, Id, Revs, Options) -> + Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_revs, + [Id, Revs, Options]), + Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, + fabric_util:recv(Workers, #shard.ref, fun handle_open_revs/3, Acc0). -update_docs(DbName, Docs, Options) -> - ok. +update_docs(DbName, AllDocs, Options) -> + GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> + Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name, Docs, Options]}), + {Shard#shard{ref=Ref}, Docs} + end, group_docs_by_shard(DbName, AllDocs)), + {Workers, _} = lists:unzip(GroupedDocs), + Acc0 = {length(Workers), length(AllDocs), couch_util:get_value(w, Options, 1), + GroupedDocs, dict:new()}, + case fabric_util:recv(Workers, #shard.ref, fun handle_update_docs/3, Acc0) of + {ok, Results} -> + {ok, couch_util:reorder_results(AllDocs, Results)}; + Else -> + Else + end. -get_missing_revs(DbName, IdsRevs) -> +get_missing_revs(_DbName, _IdsRevs) -> ok. -handle_open_doc(_Worker, {rexi_DOWN, _, _, _}, {WaitingCount, R, Replies}) -> - if WaitingCount =:= 1 -> - repair_read_quorum_failure(Replies); - true -> - {ok, {WaitingCount-1, R, Replies}} - end; -handle_open_doc(_Worker, {rexi_EXIT, Reason}, {WaitingCount, R, Replies}) -> - ?LOG_ERROR("open_doc rexi_EXIT ~p", [Reason]), - if WaitingCount =:= 1 -> - repair_read_quorum_failure(Replies); - true -> - {ok, {WaitingCount-1, R, Replies}} - end; +handle_open_doc(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_open_doc(_Worker, {rexi_EXIT, _Reason}, Acc0) -> + skip_message(Acc0); handle_open_doc(_Worker, Reply, {WaitingCount, R, Replies}) -> - ?LOG_INFO("got ~p when ~p ~p ~p", [Reply, WaitingCount, R, Replies]), - case merge_replies(make_key(Reply), Reply, Replies) of + case merge_read_reply(make_key(Reply), Reply, Replies) of {_, KeyCount} when KeyCount =:= R -> {stop, Reply}; {NewReplies, KeyCount} when KeyCount < R -> @@ -55,7 +59,82 @@ handle_open_doc(_Worker, Reply, {WaitingCount, R, Replies}) -> end end. -merge_replies(Key, Reply, Replies) -> +handle_open_revs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_open_revs(_Worker, {rexi_EXIT, _}, Acc0) -> + skip_message(Acc0); +handle_open_revs(_Worker, _Reply, {_WaitingCount, _R, _Replies}) -> + {stop, not_implemented}. + +handle_update_docs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_update_docs(_Worker, {rexi_EXIT, _}, Acc0) -> + skip_message(Acc0); +handle_update_docs(Worker, {ok, Replies}, Acc0) -> + {WaitingCount, DocCount, W, GroupedDocs, DocReplyDict0} = Acc0, + Docs = couch_util:get_value(Worker, GroupedDocs), + DocReplyDict = append_update_replies(Docs, Replies, DocReplyDict0), + case {WaitingCount, dict:size(DocReplyDict)} of + {1, _} -> + % last message has arrived, we need to conclude things + {W, Reply} = dict:fold(fun force_update_reply/3, {W,[]}, DocReplyDict), + {stop, Reply}; + {_, DocCount} -> + % we've got at least one reply for each document, let's take a look + case dict:fold(fun maybe_update_reply/3, {stop,W,[]}, DocReplyDict) of + continue -> + {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}}; + {stop, W, FinalReplies} -> + {stop, FinalReplies} + end; + {_, N} when N < DocCount -> + % no point in trying to finalize anything yet + {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}} + end. + +force_update_reply(Doc, Replies, {W, Acc}) -> + % TODO make a real decision here + case Replies of + [] -> + {W, [{Doc, {error, internal_server_error}} | Acc]}; + [Reply| _] -> + {W, [{Doc, Reply} | Acc]} + end. + +maybe_update_reply(_, _, continue) -> + % we didn't meet quorum for all docs, so we're fast-forwarding the fold + continue; +maybe_update_reply(Doc, Replies, {stop, W, Acc}) -> + case update_quorum_met(W, Replies) of + {true, Reply} -> + {stop, W, [{Doc, Reply} | Acc]}; + false -> + continue + end. + +update_quorum_met(W, Replies) -> + % TODO make a real decision here + case length(Replies) >= W of + true -> + {true, hd(Replies)}; + false -> + false + end. + +-spec group_docs_by_shard(binary(), [#doc{}]) -> [{#shard{}, [#doc{}]}]. +group_docs_by_shard(DbName, Docs) -> + dict:to_list(lists:foldl(fun(#doc{id=Id} = Doc, D0) -> + lists:foldl(fun(Shard, D1) -> + dict:append(Shard, Doc, D1) + end, D0, partitions:for_key(DbName,Id)) + end, dict:new(), Docs)). + +skip_message({1, _R, Replies}) -> + repair_read_quorum_failure(Replies); +skip_message({WaitingCount, R, Replies}) -> + {ok, {WaitingCount-1, R, Replies}}. + +merge_read_reply(Key, Reply, Replies) -> case lists:keyfind(Key, 1, Replies) of false -> {[{Key, Reply, 1} | Replies], 1}; @@ -63,6 +142,12 @@ merge_replies(Key, Reply, Replies) -> {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} end. +append_update_replies([], [], DocReplyDict) -> + DocReplyDict; +append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> + % TODO what if the same document shows up twice in one update_docs call? + append_update_replies(Rest1, Rest2, dict:append(Doc, Reply, Dict0)). + make_key({ok, #doc{id=Id, revs=Revs}}) -> {Id, Revs}; make_key({not_found, missing}) -> @@ -72,7 +157,7 @@ repair_read_quorum_failure(Replies) -> case [Doc || {ok, Doc} <- Replies] of [] -> {stop, {not_found, missing}}; - [Doc|Rest] -> + [Doc|_] -> % TODO merge docs to find the winner as determined by replication {stop, {ok, Doc}} end. diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index a0c0a568..c54ec247 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -1,10 +1,11 @@ -module(fabric_rpc). --export([open_doc/3, open_doc/4, get_db_info/1]). +-export([open_doc/3, open_doc/4, get_db_info/1, update_docs/3]). -include_lib("eunit/include/eunit.hrl"). open_doc(DbName, DocId, Options) -> - with_db(DbName, {couch_db, open_doc, [DocId, Options]}). + io:format("~p ~p ~p ~p~n", [?MODULE, DbName, DocId, Options]), + with_db(DbName, {couch_db, open_doc_int, [DocId, Options]}). %% rpc endpoints %% call to with_db will supply your M:F with a #db{} and then remaining args @@ -15,6 +16,9 @@ open_doc(DbName, DocId, Revs, Options) -> get_db_info(DbName) -> with_db(DbName, {couch_db, get_db_info, []}). +update_docs(DbName, Docs, Options) -> + with_db(DbName, {couch_db, update_docs, [Docs, Options]}). + %% %% internal %% diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 38f0b9f3..936ba4ee 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -7,6 +7,7 @@ submit_jobs(Shards, EndPoint, ExtraArgs) -> lists:map(fun(#shard{node=Node, name=ShardName} = Shard) -> + io:format("submitting ~p ~p~n", [Node, {fabric_rpc, EndPoint, [ShardName | ExtraArgs]}]), Ref = rexi:cast(Node, {fabric_rpc, EndPoint, [ShardName | ExtraArgs]}), Shard#shard{ref = Ref} end, Shards). @@ -50,6 +51,7 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> {timeout, TimeoutRef} -> timeout; {Ref, Msg} -> + io:format("process_message ~p ~p~n", [Ref, Msg]), case lists:keyfind(Ref, Keypos, RefList) of false -> % this was some non-matching message which we will ignore -- cgit v1.2.3 From a0823f1861d2b571537f75b151e936824dac068b Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 10:17:50 -0400 Subject: more forgiving interface for fabric --- src/fabric.erl | 59 +++++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index 0957387e..e0ffe16b 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,7 +1,8 @@ -module(fabric). --export([all_databases/1, create_db/2, delete_db/2, open_doc/3, open_doc/4, - get_db_info/2, db_path/2]). +-export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, + db_path/2]). +-export([open_doc/3, open_revs/4, update_docs/3]). -include("../../couch/src/couch_db.hrl"). @@ -17,26 +18,58 @@ all_databases(Customer) -> fabric_db:all_databases(Customer). get_db_info(DbName, Customer) -> - fabric_db:get_db_info(DbName, Customer). + fabric_db:get_db_info(dbname(DbName), Customer). create_db(DbName, Options) -> - fabric_db:create_db(DbName, Options). + fabric_db:create_db(dbname(DbName), Options). delete_db(DbName, Options) -> - fabric_db:delete_db(DbName, Options). + fabric_db:delete_db(dbname(DbName), Options). -% doc operations -open_doc(Db, DocId, Options) -> - fabric_doc:open_doc(Db, DocId, Options). -open_doc(Db, DocId, Revs, Options) -> - fabric_doc:open_doc(Db, DocId, Revs, Options). +open_doc(DbName, Id, Options) -> + fabric_doc:open_doc(dbname(DbName), docid(Id), Options). +open_revs(DbName, Id, Revs, Options) -> + fabric_doc:open_revs(dbname(DbName), docid(Id), Revs, Options). + +update_docs(DbName, Docs, Options) -> + fabric_doc:update_docs(dbname(DbName), docs(Docs), Options). + + +%% some simple type validation and transcoding + +dbname(DbName) when is_list(DbName) -> + list_to_binary(DbName); +dbname(DbName) when is_binary(DbName) -> + DbName; +dbname(DbName) -> + erlang:error({illegal_database_name, DbName}). + +docid(DocId) when is_list(DocId) -> + list_to_binary(DocId); +docid(DocId) when is_binary(DocId) -> + DocId; +docid(DocId) -> + erlang:error({illegal_docid, DocId}). + +docs(Docs) when is_list(Docs) -> + [doc(D) || D <- Docs]; +docs(#doc{} = Doc) -> + [Doc]; +docs({_} = Doc) -> + [couch_doc:from_json_obj(Doc)]; +docs(Docs) -> + erlang:error({illegal_docs_list, Docs}). + +doc(#doc{} = Doc) -> + Doc; +doc({_} = Doc) -> + couch_doc:from_json_obj(Doc); +doc(Doc) -> + erlang:error({illegal_doc_format, Doc}). -%% -%% internal -%% generate_customer_path("/", _Customer) -> ""; generate_customer_path("/favicon.ico", _Customer) -> -- cgit v1.2.3 From ffb5e5579a443b9c729ad52b8806dc2b83d51588 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 10:47:42 -0400 Subject: add update_doc --- src/fabric.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index e0ffe16b..3a4575bd 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,8 +1,8 @@ -module(fabric). -export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, - db_path/2]). --export([open_doc/3, open_revs/4, update_docs/3]). + db_path/2]). +-export([open_doc/3, open_revs/4, update_doc/3, update_docs/3]). -include("../../couch/src/couch_db.hrl"). @@ -34,6 +34,10 @@ open_doc(DbName, Id, Options) -> open_revs(DbName, Id, Revs, Options) -> fabric_doc:open_revs(dbname(DbName), docid(Id), Revs, Options). +update_doc(DbName, Doc, Options) -> + {ok, [Result]} = update_docs(DbName, [Doc], Options), + Result. + update_docs(DbName, Docs, Options) -> fabric_doc:update_docs(dbname(DbName), docs(Docs), Options). @@ -56,10 +60,6 @@ docid(DocId) -> docs(Docs) when is_list(Docs) -> [doc(D) || D <- Docs]; -docs(#doc{} = Doc) -> - [Doc]; -docs({_} = Doc) -> - [couch_doc:from_json_obj(Doc)]; docs(Docs) -> erlang:error({illegal_docs_list, Docs}). -- cgit v1.2.3 From 58ce4b99b19affa3aaf335b6dc257301ef813c69 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 10:48:34 -0400 Subject: fix matching of open_doc replies --- src/fabric_doc.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fabric_doc.erl b/src/fabric_doc.erl index c3cc0f47..039b6aab 100644 --- a/src/fabric_doc.erl +++ b/src/fabric_doc.erl @@ -10,12 +10,12 @@ open_doc(DbName, Id, Options) -> SuppressDeletedDoc = not lists:member(deleted, Options), Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, case fabric_util:recv(Workers, #shard.ref, fun handle_open_doc/3, Acc0) of - {ok, #doc{deleted=true}} when SuppressDeletedDoc -> + {ok, {ok, #doc{deleted=true}}} when SuppressDeletedDoc -> {not_found, deleted}; - {ok, {not_found, missing}} -> - {not_found, missing}; - Else -> - Else + {ok, Else} -> + Else; + Error -> + Error end. open_revs(DbName, Id, Revs, Options) -> @@ -154,7 +154,7 @@ make_key({not_found, missing}) -> {not_found, missing}. repair_read_quorum_failure(Replies) -> - case [Doc || {ok, Doc} <- Replies] of + case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of [] -> {stop, {not_found, missing}}; [Doc|_] -> -- cgit v1.2.3 From 8dc112522f8baef4454f3c0ca74fbbd920fc451d Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Fri, 28 May 2010 11:42:43 -0400 Subject: always install fullmap during create_db, and make create/delete db calls use #shard{name=ShardName} --- src/fabric_db.erl | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/fabric_db.erl b/src/fabric_db.erl index 95648e01..088faa7b 100644 --- a/src/fabric_db.erl +++ b/src/fabric_db.erl @@ -45,26 +45,27 @@ get_db_info(DbName, Customer) -> create_db(DbName, Options) -> Fullmap = partitions:fullmap(DbName, Options), {ok, FullNodes} = mem3:fullnodes(), - RefPartMap = send_create_calls(DbName, Options, Fullmap), + RefPartMap = send_create_calls(Fullmap, Options), Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || {_,#shard{range=[Beg,_]}} <- RefPartMap])}, - case fabric_util:receive_loop( - RefPartMap, 1, fun handle_create_msg/3, Acc0, 5000, infinity) of - {ok, _Results} -> - partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), - ok; + Result = case fabric_util:receive_loop( + RefPartMap, 1, fun handle_create_msg/3, Acc0) of + {ok, _Results} -> ok; Error -> Error - end. + end, + % always install partition map, even w/ errors, so delete is possible + partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), + Result. %% @doc Delete a database, and all its partition files across the cluster %% Options is proplist with user_ctx, n, q -spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. delete_db(DbName, Options) -> - Parts = partitions:all_parts(DbName), - RefPartMap = send_delete_calls(DbName, Options, Parts), + Fullmap = partitions:all_parts(DbName), + RefPartMap = send_delete_calls(Fullmap, Options), Acc0 = {true, length(RefPartMap)}, case fabric_util:receive_loop( - RefPartMap, 1, fun handle_delete_msg/3, Acc0, 5000, infinity) of + RefPartMap, 1, fun handle_delete_msg/3, Acc0) of {ok, _Results} -> delete_fullmap(DbName), ok; @@ -152,10 +153,9 @@ max(New, Existing) -> %% @doc create the partitions on all appropriate nodes (rexi calls) --spec send_create_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. -send_create_calls(DbName, Options, Fullmap) -> - lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> - ShardName = showroom_utils:shard_name(Beg, DbName), +-spec send_create_calls(fullmap(), list()) -> [{reference(), part()}]. +send_create_calls(Fullmap, Options) -> + lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> Ref = rexi:async_server_call({couch_server, Node}, {create, ShardName, Options}), {Ref, Part} @@ -186,10 +186,9 @@ handle_create_msg({_Ref, #shard{range=[Beg,_]}}, {ok, _}, {false, Rem, PartResul %% @doc delete the partitions on all appropriate nodes (rexi calls) --spec send_delete_calls(binary(), list(), fullmap()) -> [{reference(), part()}]. -send_delete_calls(DbName, Options, Parts) -> - lists:map(fun(#shard{node=Node, range=[Beg,_]} = Part) -> - ShardName = showroom_utils:shard_name(Beg, DbName), +-spec send_delete_calls(fullmap(), list()) -> [{reference(), part()}]. +send_delete_calls(Parts, Options) -> + lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> Ref = rexi:async_server_call({couch_server, Node}, {delete, ShardName, Options}), {Ref, Part} -- cgit v1.2.3 From 450bc69ea891d9ad03c26c31c181c28ee5cbe8f9 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 13:15:38 -0400 Subject: implement fabric missing_revs, BugzID 10217 --- src/fabric.erl | 15 ++++++++++++++- src/fabric_doc.erl | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++--- src/fabric_rpc.erl | 23 ++++++++++++++++++++-- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index 3a4575bd..26b379af 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -2,7 +2,8 @@ -export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, db_path/2]). --export([open_doc/3, open_revs/4, update_doc/3, update_docs/3]). +-export([open_doc/3, open_revs/4, get_missing_revs/2]). +-export([update_doc/3, update_docs/3]). -include("../../couch/src/couch_db.hrl"). @@ -34,6 +35,10 @@ open_doc(DbName, Id, Options) -> open_revs(DbName, Id, Revs, Options) -> fabric_doc:open_revs(dbname(DbName), docid(Id), Revs, Options). +get_missing_revs(DbName, IdsRevs) when is_list(IdsRevs) -> + Sanitized = [idrevs(IdR) || IdR <- IdsRevs], + fabric_doc:get_missing_revs(dbname(DbName), Sanitized). + update_doc(DbName, Doc, Options) -> {ok, [Result]} = update_docs(DbName, [Doc], Options), Result. @@ -70,6 +75,14 @@ doc({_} = Doc) -> doc(Doc) -> erlang:error({illegal_doc_format, Doc}). +idrevs({Id, Revs}) when is_list(Revs) -> + {docid(Id), [rev(R) || R <- Revs]}. + +rev(Rev) when is_list(Rev); is_binary(Rev) -> + couch_doc:parse_rev(Rev); +rev({Seq, Hash} = Rev) when is_integer(Seq), is_binary(Hash) -> + Rev. + generate_customer_path("/", _Customer) -> ""; generate_customer_path("/favicon.ico", _Customer) -> diff --git a/src/fabric_doc.erl b/src/fabric_doc.erl index 039b6aab..97e7cfb6 100644 --- a/src/fabric_doc.erl +++ b/src/fabric_doc.erl @@ -24,6 +24,15 @@ open_revs(DbName, Id, Revs, Options) -> Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, fabric_util:recv(Workers, #shard.ref, fun handle_open_revs/3, Acc0). +get_missing_revs(DbName, AllIdsRevs) -> + Workers = lists:map(fun({#shard{name=Name, node=Node} = Shard, IdsRevs}) -> + Ref = rexi:cast(Node, {fabric_rpc, get_missing_revs, [Name, IdsRevs]}), + Shard#shard{ref=Ref} + end, group_idrevs_by_shard(DbName, AllIdsRevs)), + ResultDict = dict:from_list([{Id, {nil,Revs}} || {Id, Revs} <- AllIdsRevs]), + Acc0 = {length(Workers), ResultDict}, + fabric_util:recv(Workers, #shard.ref, fun handle_missing_revs/3, Acc0). + update_docs(DbName, AllDocs, Options) -> GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name, Docs, Options]}), @@ -39,9 +48,6 @@ update_docs(DbName, AllDocs, Options) -> Else end. -get_missing_revs(_DbName, _IdsRevs) -> - ok. - handle_open_doc(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> skip_message(Acc0); handle_open_doc(_Worker, {rexi_EXIT, _Reason}, Acc0) -> @@ -66,6 +72,23 @@ handle_open_revs(_Worker, {rexi_EXIT, _}, Acc0) -> handle_open_revs(_Worker, _Reply, {_WaitingCount, _R, _Replies}) -> {stop, not_implemented}. +handle_missing_revs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_missing_revs(_Worker, {rexi_EXIT, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_missing_revs(_Worker, {ok, Results}, {1, D0}) -> + D = update_dict(D0, Results), + {stop, dict:fold(fun force_missing_revs_reply/3, [], D)}; +handle_missing_revs(_Worker, {ok, Results}, {WaitingCount, D0}) -> + D = update_dict(D0, Results), + case dict:fold(fun maybe_missing_revs_reply/3, {stop, []}, D) of + continue -> + % still haven't heard about some Ids + {ok, {WaitingCount - 1, D}}; + {stop, FinalReply} -> + {stop, FinalReply} + end. + handle_update_docs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> skip_message(Acc0); handle_update_docs(_Worker, {rexi_EXIT, _}, Acc0) -> @@ -92,6 +115,23 @@ handle_update_docs(Worker, {ok, Replies}, Acc0) -> {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}} end. +force_missing_revs_reply(Id, {nil,Revs}, Acc) -> + % never heard about this ID, assume it's missing + [{Id, Revs} | Acc]; +force_missing_revs_reply(_, [], Acc) -> + Acc; +force_missing_revs_reply(Id, Revs, Acc) -> + [{Id, Revs} | Acc]. + +maybe_missing_revs_reply(_, _, continue) -> + continue; +maybe_missing_revs_reply(_, {nil, _}, _) -> + continue; +maybe_missing_revs_reply(_, [], {stop, Acc}) -> + {stop, Acc}; +maybe_missing_revs_reply(Id, Revs, {stop, Acc}) -> + {stop, [{Id, Revs} | Acc]}. + force_update_reply(Doc, Replies, {W, Acc}) -> % TODO make a real decision here case Replies of @@ -129,6 +169,13 @@ group_docs_by_shard(DbName, Docs) -> end, D0, partitions:for_key(DbName,Id)) end, dict:new(), Docs)). +group_idrevs_by_shard(DbName, IdsRevs) -> + dict:to_list(lists:foldl(fun({Id, Revs}, D0) -> + lists:foldl(fun(Shard, D1) -> + dict:append(Shard, {Id, Revs}, D1) + end, D0, partitions:for_key(DbName,Id)) + end, dict:new(), IdsRevs)). + skip_message({1, _R, Replies}) -> repair_read_quorum_failure(Replies); skip_message({WaitingCount, R, Replies}) -> @@ -142,6 +189,9 @@ merge_read_reply(Key, Reply, Replies) -> {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} end. +update_dict(D0, KVs) -> + lists:foldl(fun({K,V}, D1) -> dict:store(K, V, D1) end, D0, KVs). + append_update_replies([], [], DocReplyDict) -> DocReplyDict; append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index c54ec247..75ce5e90 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -1,12 +1,31 @@ -module(fabric_rpc). --export([open_doc/3, open_doc/4, get_db_info/1, update_docs/3]). +-export([open_doc/3, open_doc/4, get_db_info/1, update_docs/3, + get_missing_revs/2]). +-include("../../couch/src/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). + open_doc(DbName, DocId, Options) -> - io:format("~p ~p ~p ~p~n", [?MODULE, DbName, DocId, Options]), with_db(DbName, {couch_db, open_doc_int, [DocId, Options]}). +get_missing_revs(DbName, IdRevsList) -> + % reimplement here so we get [] for Ids with no missing revs in response + rexi:reply(case couch_db:open(DbName, []) of + {ok, Db} -> + Ids = [Id1 || {Id1, _Revs} <- IdRevsList], + {ok, lists:zipwith(fun({Id, Revs}, FullDocInfoResult) -> + case FullDocInfoResult of + {ok, #full_doc_info{rev_tree=RevisionTree}} -> + {Id, couch_key_tree:find_missing(RevisionTree, Revs)}; + not_found -> + {Id, Revs} + end + end, IdRevsList, couch_btree:lookup(Db#db.id_tree, Ids))}; + Error -> + Error + end). + %% rpc endpoints %% call to with_db will supply your M:F with a #db{} and then remaining args -- cgit v1.2.3 From 12318c9384bd8f8d82b0c1501e3e8ab0fea34095 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 14:26:07 -0400 Subject: open_revs for fabric, BugzID 10216 --- src/fabric_doc.erl | 25 ++++++++++++++++++++----- src/fabric_rpc.erl | 12 ++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/fabric_doc.erl b/src/fabric_doc.erl index 97e7cfb6..e641d286 100644 --- a/src/fabric_doc.erl +++ b/src/fabric_doc.erl @@ -22,7 +22,12 @@ open_revs(DbName, Id, Revs, Options) -> Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_revs, [Id, Revs, Options]), Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, - fabric_util:recv(Workers, #shard.ref, fun handle_open_revs/3, Acc0). + case fabric_util:recv(Workers, #shard.ref, fun handle_open_revs/3, Acc0) of + {ok, {ok, Reply}} -> + {ok, Reply}; + Else -> + Else + end. get_missing_revs(DbName, AllIdsRevs) -> Workers = lists:map(fun({#shard{name=Name, node=Node} = Shard, IdsRevs}) -> @@ -69,8 +74,18 @@ handle_open_revs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> skip_message(Acc0); handle_open_revs(_Worker, {rexi_EXIT, _}, Acc0) -> skip_message(Acc0); -handle_open_revs(_Worker, _Reply, {_WaitingCount, _R, _Replies}) -> - {stop, not_implemented}. +handle_open_revs(_Worker, Reply, {WaitingCount, R, Replies}) -> + case merge_read_reply(make_key(Reply), Reply, Replies) of + {_, KeyCount} when KeyCount =:= R -> + {stop, Reply}; + {NewReplies, KeyCount} when KeyCount < R -> + if WaitingCount =:= 1 -> + % last message arrived, but still no quorum + repair_read_quorum_failure(NewReplies); + true -> + {ok, {WaitingCount-1, R, NewReplies}} + end + end. handle_missing_revs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> skip_message(Acc0); @@ -200,8 +215,8 @@ append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> make_key({ok, #doc{id=Id, revs=Revs}}) -> {Id, Revs}; -make_key({not_found, missing}) -> - {not_found, missing}. +make_key(Else) -> + Else. repair_read_quorum_failure(Replies) -> case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 75ce5e90..90ee8627 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -1,7 +1,7 @@ -module(fabric_rpc). --export([open_doc/3, open_doc/4, get_db_info/1, update_docs/3, - get_missing_revs/2]). +-export([get_db_info/1]). +-export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). -include("../../couch/src/couch_db.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -25,13 +25,13 @@ get_missing_revs(DbName, IdRevsList) -> Error -> Error end). - + +open_revs(DbName, Id, Revs, Options) -> + with_db(DbName, {couch_db, open_doc_revs, [Id, Revs, Options]}). + %% rpc endpoints %% call to with_db will supply your M:F with a #db{} and then remaining args -open_doc(DbName, DocId, Revs, Options) -> - with_db(DbName, {couch_api, open_doc, [DocId, Revs, Options]}). - get_db_info(DbName) -> with_db(DbName, {couch_db, get_db_info, []}). -- cgit v1.2.3 From 5f8844cba7f55ed416050c2bdb8ee8a554a6f191 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 14:54:39 -0400 Subject: split fabric_doc into dedicated modules --- ebin/fabric.app | 4 + include/fabric.hrl | 2 + src/fabric.erl | 8 +- src/fabric_doc.erl | 228 -------------------------------------------- src/fabric_missing_revs.erl | 61 ++++++++++++ src/fabric_open_doc.erl | 61 ++++++++++++ src/fabric_open_revs.erl | 60 ++++++++++++ src/fabric_update_docs.erl | 92 ++++++++++++++++++ 8 files changed, 284 insertions(+), 232 deletions(-) create mode 100644 include/fabric.hrl delete mode 100644 src/fabric_doc.erl create mode 100644 src/fabric_missing_revs.erl create mode 100644 src/fabric_open_doc.erl create mode 100644 src/fabric_open_revs.erl create mode 100644 src/fabric_update_docs.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index e08f560b..363655b2 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -7,7 +7,11 @@ fabric, fabric_db, fabric_doc, + fabric_missing_revs, + fabric_open_doc, + fabric_open_revs, fabric_rpc, + fabric_update_docs, fabric_util ]}, {registered, []}, diff --git a/include/fabric.hrl b/include/fabric.hrl new file mode 100644 index 00000000..5426addc --- /dev/null +++ b/include/fabric.hrl @@ -0,0 +1,2 @@ +-include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). diff --git a/src/fabric.erl b/src/fabric.erl index 26b379af..399b76d6 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -30,21 +30,21 @@ delete_db(DbName, Options) -> open_doc(DbName, Id, Options) -> - fabric_doc:open_doc(dbname(DbName), docid(Id), Options). + fabric_open_doc:go(dbname(DbName), docid(Id), Options). open_revs(DbName, Id, Revs, Options) -> - fabric_doc:open_revs(dbname(DbName), docid(Id), Revs, Options). + fabric_open_revs:go(dbname(DbName), docid(Id), Revs, Options). get_missing_revs(DbName, IdsRevs) when is_list(IdsRevs) -> Sanitized = [idrevs(IdR) || IdR <- IdsRevs], - fabric_doc:get_missing_revs(dbname(DbName), Sanitized). + fabric_missing_revs:go(dbname(DbName), Sanitized). update_doc(DbName, Doc, Options) -> {ok, [Result]} = update_docs(DbName, [Doc], Options), Result. update_docs(DbName, Docs, Options) -> - fabric_doc:update_docs(dbname(DbName), docs(Docs), Options). + fabric_update_docs:go(dbname(DbName), docs(Docs), Options). %% some simple type validation and transcoding diff --git a/src/fabric_doc.erl b/src/fabric_doc.erl deleted file mode 100644 index e641d286..00000000 --- a/src/fabric_doc.erl +++ /dev/null @@ -1,228 +0,0 @@ --module(fabric_doc). --export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). - --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - -open_doc(DbName, Id, Options) -> - Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_doc, - [Id, Options]), - SuppressDeletedDoc = not lists:member(deleted, Options), - Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, - case fabric_util:recv(Workers, #shard.ref, fun handle_open_doc/3, Acc0) of - {ok, {ok, #doc{deleted=true}}} when SuppressDeletedDoc -> - {not_found, deleted}; - {ok, Else} -> - Else; - Error -> - Error - end. - -open_revs(DbName, Id, Revs, Options) -> - Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_revs, - [Id, Revs, Options]), - Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, - case fabric_util:recv(Workers, #shard.ref, fun handle_open_revs/3, Acc0) of - {ok, {ok, Reply}} -> - {ok, Reply}; - Else -> - Else - end. - -get_missing_revs(DbName, AllIdsRevs) -> - Workers = lists:map(fun({#shard{name=Name, node=Node} = Shard, IdsRevs}) -> - Ref = rexi:cast(Node, {fabric_rpc, get_missing_revs, [Name, IdsRevs]}), - Shard#shard{ref=Ref} - end, group_idrevs_by_shard(DbName, AllIdsRevs)), - ResultDict = dict:from_list([{Id, {nil,Revs}} || {Id, Revs} <- AllIdsRevs]), - Acc0 = {length(Workers), ResultDict}, - fabric_util:recv(Workers, #shard.ref, fun handle_missing_revs/3, Acc0). - -update_docs(DbName, AllDocs, Options) -> - GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> - Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name, Docs, Options]}), - {Shard#shard{ref=Ref}, Docs} - end, group_docs_by_shard(DbName, AllDocs)), - {Workers, _} = lists:unzip(GroupedDocs), - Acc0 = {length(Workers), length(AllDocs), couch_util:get_value(w, Options, 1), - GroupedDocs, dict:new()}, - case fabric_util:recv(Workers, #shard.ref, fun handle_update_docs/3, Acc0) of - {ok, Results} -> - {ok, couch_util:reorder_results(AllDocs, Results)}; - Else -> - Else - end. - -handle_open_doc(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> - skip_message(Acc0); -handle_open_doc(_Worker, {rexi_EXIT, _Reason}, Acc0) -> - skip_message(Acc0); -handle_open_doc(_Worker, Reply, {WaitingCount, R, Replies}) -> - case merge_read_reply(make_key(Reply), Reply, Replies) of - {_, KeyCount} when KeyCount =:= R -> - {stop, Reply}; - {NewReplies, KeyCount} when KeyCount < R -> - if WaitingCount =:= 1 -> - % last message arrived, but still no quorum - repair_read_quorum_failure(NewReplies); - true -> - {ok, {WaitingCount-1, R, NewReplies}} - end - end. - -handle_open_revs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> - skip_message(Acc0); -handle_open_revs(_Worker, {rexi_EXIT, _}, Acc0) -> - skip_message(Acc0); -handle_open_revs(_Worker, Reply, {WaitingCount, R, Replies}) -> - case merge_read_reply(make_key(Reply), Reply, Replies) of - {_, KeyCount} when KeyCount =:= R -> - {stop, Reply}; - {NewReplies, KeyCount} when KeyCount < R -> - if WaitingCount =:= 1 -> - % last message arrived, but still no quorum - repair_read_quorum_failure(NewReplies); - true -> - {ok, {WaitingCount-1, R, NewReplies}} - end - end. - -handle_missing_revs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> - skip_message(Acc0); -handle_missing_revs(_Worker, {rexi_EXIT, _, _, _}, Acc0) -> - skip_message(Acc0); -handle_missing_revs(_Worker, {ok, Results}, {1, D0}) -> - D = update_dict(D0, Results), - {stop, dict:fold(fun force_missing_revs_reply/3, [], D)}; -handle_missing_revs(_Worker, {ok, Results}, {WaitingCount, D0}) -> - D = update_dict(D0, Results), - case dict:fold(fun maybe_missing_revs_reply/3, {stop, []}, D) of - continue -> - % still haven't heard about some Ids - {ok, {WaitingCount - 1, D}}; - {stop, FinalReply} -> - {stop, FinalReply} - end. - -handle_update_docs(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> - skip_message(Acc0); -handle_update_docs(_Worker, {rexi_EXIT, _}, Acc0) -> - skip_message(Acc0); -handle_update_docs(Worker, {ok, Replies}, Acc0) -> - {WaitingCount, DocCount, W, GroupedDocs, DocReplyDict0} = Acc0, - Docs = couch_util:get_value(Worker, GroupedDocs), - DocReplyDict = append_update_replies(Docs, Replies, DocReplyDict0), - case {WaitingCount, dict:size(DocReplyDict)} of - {1, _} -> - % last message has arrived, we need to conclude things - {W, Reply} = dict:fold(fun force_update_reply/3, {W,[]}, DocReplyDict), - {stop, Reply}; - {_, DocCount} -> - % we've got at least one reply for each document, let's take a look - case dict:fold(fun maybe_update_reply/3, {stop,W,[]}, DocReplyDict) of - continue -> - {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}}; - {stop, W, FinalReplies} -> - {stop, FinalReplies} - end; - {_, N} when N < DocCount -> - % no point in trying to finalize anything yet - {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}} - end. - -force_missing_revs_reply(Id, {nil,Revs}, Acc) -> - % never heard about this ID, assume it's missing - [{Id, Revs} | Acc]; -force_missing_revs_reply(_, [], Acc) -> - Acc; -force_missing_revs_reply(Id, Revs, Acc) -> - [{Id, Revs} | Acc]. - -maybe_missing_revs_reply(_, _, continue) -> - continue; -maybe_missing_revs_reply(_, {nil, _}, _) -> - continue; -maybe_missing_revs_reply(_, [], {stop, Acc}) -> - {stop, Acc}; -maybe_missing_revs_reply(Id, Revs, {stop, Acc}) -> - {stop, [{Id, Revs} | Acc]}. - -force_update_reply(Doc, Replies, {W, Acc}) -> - % TODO make a real decision here - case Replies of - [] -> - {W, [{Doc, {error, internal_server_error}} | Acc]}; - [Reply| _] -> - {W, [{Doc, Reply} | Acc]} - end. - -maybe_update_reply(_, _, continue) -> - % we didn't meet quorum for all docs, so we're fast-forwarding the fold - continue; -maybe_update_reply(Doc, Replies, {stop, W, Acc}) -> - case update_quorum_met(W, Replies) of - {true, Reply} -> - {stop, W, [{Doc, Reply} | Acc]}; - false -> - continue - end. - -update_quorum_met(W, Replies) -> - % TODO make a real decision here - case length(Replies) >= W of - true -> - {true, hd(Replies)}; - false -> - false - end. - --spec group_docs_by_shard(binary(), [#doc{}]) -> [{#shard{}, [#doc{}]}]. -group_docs_by_shard(DbName, Docs) -> - dict:to_list(lists:foldl(fun(#doc{id=Id} = Doc, D0) -> - lists:foldl(fun(Shard, D1) -> - dict:append(Shard, Doc, D1) - end, D0, partitions:for_key(DbName,Id)) - end, dict:new(), Docs)). - -group_idrevs_by_shard(DbName, IdsRevs) -> - dict:to_list(lists:foldl(fun({Id, Revs}, D0) -> - lists:foldl(fun(Shard, D1) -> - dict:append(Shard, {Id, Revs}, D1) - end, D0, partitions:for_key(DbName,Id)) - end, dict:new(), IdsRevs)). - -skip_message({1, _R, Replies}) -> - repair_read_quorum_failure(Replies); -skip_message({WaitingCount, R, Replies}) -> - {ok, {WaitingCount-1, R, Replies}}. - -merge_read_reply(Key, Reply, Replies) -> - case lists:keyfind(Key, 1, Replies) of - false -> - {[{Key, Reply, 1} | Replies], 1}; - {Key, _, N} -> - {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} - end. - -update_dict(D0, KVs) -> - lists:foldl(fun({K,V}, D1) -> dict:store(K, V, D1) end, D0, KVs). - -append_update_replies([], [], DocReplyDict) -> - DocReplyDict; -append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> - % TODO what if the same document shows up twice in one update_docs call? - append_update_replies(Rest1, Rest2, dict:append(Doc, Reply, Dict0)). - -make_key({ok, #doc{id=Id, revs=Revs}}) -> - {Id, Revs}; -make_key(Else) -> - Else. - -repair_read_quorum_failure(Replies) -> - case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of - [] -> - {stop, {not_found, missing}}; - [Doc|_] -> - % TODO merge docs to find the winner as determined by replication - {stop, {ok, Doc}} - end. diff --git a/src/fabric_missing_revs.erl b/src/fabric_missing_revs.erl new file mode 100644 index 00000000..d329d2aa --- /dev/null +++ b/src/fabric_missing_revs.erl @@ -0,0 +1,61 @@ +-module(fabric_missing_revs). +-export([go/2]). +-include("fabric.hrl"). + +go(DbName, AllIdsRevs) -> + Workers = lists:map(fun({#shard{name=Name, node=Node} = Shard, IdsRevs}) -> + Ref = rexi:cast(Node, {fabric_rpc, get_missing_revs, [Name, IdsRevs]}), + Shard#shard{ref=Ref} + end, group_idrevs_by_shard(DbName, AllIdsRevs)), + ResultDict = dict:from_list([{Id, {nil,Revs}} || {Id, Revs} <- AllIdsRevs]), + Acc0 = {length(Workers), ResultDict}, + fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). + +handle_message(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_message(_Worker, {rexi_EXIT, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_message(_Worker, {ok, Results}, {1, D0}) -> + D = update_dict(D0, Results), + {stop, dict:fold(fun force_reply/3, [], D)}; +handle_message(_Worker, {ok, Results}, {WaitingCount, D0}) -> + D = update_dict(D0, Results), + case dict:fold(fun maybe_reply/3, {stop, []}, D) of + continue -> + % still haven't heard about some Ids + {ok, {WaitingCount - 1, D}}; + {stop, FinalReply} -> + {stop, FinalReply} + end. + +force_reply(Id, {nil,Revs}, Acc) -> + % never heard about this ID, assume it's missing + [{Id, Revs} | Acc]; +force_reply(_, [], Acc) -> + Acc; +force_reply(Id, Revs, Acc) -> + [{Id, Revs} | Acc]. + +maybe_reply(_, _, continue) -> + continue; +maybe_reply(_, {nil, _}, _) -> + continue; +maybe_reply(_, [], {stop, Acc}) -> + {stop, Acc}; +maybe_reply(Id, Revs, {stop, Acc}) -> + {stop, [{Id, Revs} | Acc]}. + +group_idrevs_by_shard(DbName, IdsRevs) -> + dict:to_list(lists:foldl(fun({Id, Revs}, D0) -> + lists:foldl(fun(Shard, D1) -> + dict:append(Shard, {Id, Revs}, D1) + end, D0, partitions:for_key(DbName,Id)) + end, dict:new(), IdsRevs)). + +update_dict(D0, KVs) -> + lists:foldl(fun({K,V}, D1) -> dict:store(K, V, D1) end, D0, KVs). + +skip_message({1, Dict}) -> + {stop, dict:fold(fun force_reply/3, [], Dict)}; +skip_message({WaitingCount, Dict}) -> + {ok, {WaitingCount-1, Dict}}. diff --git a/src/fabric_open_doc.erl b/src/fabric_open_doc.erl new file mode 100644 index 00000000..996282d8 --- /dev/null +++ b/src/fabric_open_doc.erl @@ -0,0 +1,61 @@ +-module(fabric_open_doc). +-export([go/3]). +-include("fabric.hrl"). + +go(DbName, Id, Options) -> + Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_doc, + [Id, Options]), + SuppressDeletedDoc = not lists:member(deleted, Options), + Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, {ok, #doc{deleted=true}}} when SuppressDeletedDoc -> + {not_found, deleted}; + {ok, Else} -> + Else; + Error -> + Error + end. + +handle_message(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_message(_Worker, {rexi_EXIT, _Reason}, Acc0) -> + skip_message(Acc0); +handle_message(_Worker, Reply, {WaitingCount, R, Replies}) -> + case merge_read_reply(make_key(Reply), Reply, Replies) of + {_, KeyCount} when KeyCount =:= R -> + {stop, Reply}; + {NewReplies, KeyCount} when KeyCount < R -> + if WaitingCount =:= 1 -> + % last message arrived, but still no quorum + repair_read_quorum_failure(NewReplies); + true -> + {ok, {WaitingCount-1, R, NewReplies}} + end + end. + +skip_message({1, _R, Replies}) -> + repair_read_quorum_failure(Replies); +skip_message({WaitingCount, R, Replies}) -> + {ok, {WaitingCount-1, R, Replies}}. + +merge_read_reply(Key, Reply, Replies) -> + case lists:keyfind(Key, 1, Replies) of + false -> + {[{Key, Reply, 1} | Replies], 1}; + {Key, _, N} -> + {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} + end. + +make_key({ok, #doc{id=Id, revs=Revs}}) -> + {Id, Revs}; +make_key(Else) -> + Else. + +repair_read_quorum_failure(Replies) -> + case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of + [] -> + {stop, {not_found, missing}}; + [Doc|_] -> + % TODO merge docs to find the winner as determined by replication + {stop, {ok, Doc}} + end. \ No newline at end of file diff --git a/src/fabric_open_revs.erl b/src/fabric_open_revs.erl new file mode 100644 index 00000000..c5bd586d --- /dev/null +++ b/src/fabric_open_revs.erl @@ -0,0 +1,60 @@ +-module(fabric_open_revs). +-export([go/4]). +-include("fabric.hrl"). + +go(DbName, Id, Revs, Options) -> + Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_revs, + [Id, Revs, Options]), + Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, {ok, Reply}} -> + {ok, Reply}; + Else -> + Else + end. + +handle_message(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_message(_Worker, {rexi_EXIT, _}, Acc0) -> + skip_message(Acc0); +handle_message(_Worker, Reply, {WaitingCount, R, Replies}) -> + case merge_read_reply(make_key(Reply), Reply, Replies) of + {_, KeyCount} when KeyCount =:= R -> + {stop, Reply}; + {NewReplies, KeyCount} when KeyCount < R -> + if WaitingCount =:= 1 -> + % last message arrived, but still no quorum + repair_read_quorum_failure(NewReplies); + true -> + {ok, {WaitingCount-1, R, NewReplies}} + end + end. + +skip_message({1, _R, Replies}) -> + repair_read_quorum_failure(Replies); +skip_message({WaitingCount, R, Replies}) -> + {ok, {WaitingCount-1, R, Replies}}. + +merge_read_reply(Key, Reply, Replies) -> + case lists:keyfind(Key, 1, Replies) of + false -> + {[{Key, Reply, 1} | Replies], 1}; + {Key, _, N} -> + {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} + end. + +make_key({ok, #doc{id=Id, revs=Revs}}) -> + {Id, Revs}; +make_key(Else) -> + Else. + +repair_read_quorum_failure(Replies) -> + case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of + [] -> + {stop, {not_found, missing}}; + [Doc|_] -> + % TODO merge docs to find the winner as determined by replication + {stop, {ok, Doc}} + end. + + \ No newline at end of file diff --git a/src/fabric_update_docs.erl b/src/fabric_update_docs.erl new file mode 100644 index 00000000..00c7e9d7 --- /dev/null +++ b/src/fabric_update_docs.erl @@ -0,0 +1,92 @@ +-module(fabric_update_docs). +-export([go/3]). +-include("fabric.hrl"). + +go(DbName, AllDocs, Options) -> + GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> + Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name, Docs, Options]}), + {Shard#shard{ref=Ref}, Docs} + end, group_docs_by_shard(DbName, AllDocs)), + {Workers, _} = lists:unzip(GroupedDocs), + Acc0 = {length(Workers), length(AllDocs), couch_util:get_value(w, Options, 1), + GroupedDocs, dict:new()}, + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, Results} -> + {ok, couch_util:reorder_results(AllDocs, Results)}; + Else -> + Else + end. + +handle_message(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> + skip_message(Acc0); +handle_message(_Worker, {rexi_EXIT, _}, Acc0) -> + skip_message(Acc0); +handle_message(Worker, {ok, Replies}, Acc0) -> + {WaitingCount, DocCount, W, GroupedDocs, DocReplyDict0} = Acc0, + Docs = couch_util:get_value(Worker, GroupedDocs), + DocReplyDict = append_update_replies(Docs, Replies, DocReplyDict0), + case {WaitingCount, dict:size(DocReplyDict)} of + {1, _} -> + % last message has arrived, we need to conclude things + {W, Reply} = dict:fold(fun force_reply/3, {W,[]}, DocReplyDict), + {stop, Reply}; + {_, DocCount} -> + % we've got at least one reply for each document, let's take a look + case dict:fold(fun maybe_reply/3, {stop,W,[]}, DocReplyDict) of + continue -> + {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}}; + {stop, W, FinalReplies} -> + {stop, FinalReplies} + end; + {_, N} when N < DocCount -> + % no point in trying to finalize anything yet + {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}} + end. + +force_reply(Doc, Replies, {W, Acc}) -> + % TODO make a real decision here + case Replies of + [] -> + {W, [{Doc, {error, internal_server_error}} | Acc]}; + [Reply| _] -> + {W, [{Doc, Reply} | Acc]} + end. + +maybe_reply(_, _, continue) -> + % we didn't meet quorum for all docs, so we're fast-forwarding the fold + continue; +maybe_reply(Doc, Replies, {stop, W, Acc}) -> + case update_quorum_met(W, Replies) of + {true, Reply} -> + {stop, W, [{Doc, Reply} | Acc]}; + false -> + continue + end. + +update_quorum_met(W, Replies) -> + % TODO make a real decision here + case length(Replies) >= W of + true -> + {true, hd(Replies)}; + false -> + false + end. + +-spec group_docs_by_shard(binary(), [#doc{}]) -> [{#shard{}, [#doc{}]}]. +group_docs_by_shard(DbName, Docs) -> + dict:to_list(lists:foldl(fun(#doc{id=Id} = Doc, D0) -> + lists:foldl(fun(Shard, D1) -> + dict:append(Shard, Doc, D1) + end, D0, partitions:for_key(DbName,Id)) + end, dict:new(), Docs)). + +append_update_replies([], [], DocReplyDict) -> + DocReplyDict; +append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> + % TODO what if the same document shows up twice in one update_docs call? + append_update_replies(Rest1, Rest2, dict:append(Doc, Reply, Dict0)). + +skip_message(Acc0) -> + % TODO fix this + {ok, Acc0}. + -- cgit v1.2.3 From 46273631d3496586bfb1fa1713c2e9b8484bec61 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 15:21:42 -0400 Subject: reorganize rpc calls, add doc_count and update_seq --- src/fabric_rpc.erl | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 90ee8627..fa67b13f 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -1,14 +1,40 @@ -module(fabric_rpc). --export([get_db_info/1]). +-export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). --include("../../couch/src/couch_db.hrl"). +-include("fabric.hrl"). -include_lib("eunit/include/eunit.hrl"). +%% rpc endpoints +%% call to with_db will supply your M:F with a #db{} and then remaining args + +get_db_info(DbName) -> + with_db(DbName, {couch_db, get_db_info, []}). + +get_doc_count(DbName) -> + rexi:reply(case couch_db:open(DbName) of + {ok, Db} -> + {ok, {Count, _DelCount}} = couch_btree:full_reduce(Db#db.id_tree), + {ok, Count}; + Error -> + Error + end). + +get_update_seq(DbName) -> + rexi:reply(case couch_db:open(DbName) of + {ok, #db{update_seq = Seq}} -> + {ok, Seq}; + Error -> + Error + end). + open_doc(DbName, DocId, Options) -> with_db(DbName, {couch_db, open_doc_int, [DocId, Options]}). +open_revs(DbName, Id, Revs, Options) -> + with_db(DbName, {couch_db, open_doc_revs, [Id, Revs, Options]}). + get_missing_revs(DbName, IdRevsList) -> % reimplement here so we get [] for Ids with no missing revs in response rexi:reply(case couch_db:open(DbName, []) of @@ -26,15 +52,6 @@ get_missing_revs(DbName, IdRevsList) -> Error end). -open_revs(DbName, Id, Revs, Options) -> - with_db(DbName, {couch_db, open_doc_revs, [Id, Revs, Options]}). - -%% rpc endpoints -%% call to with_db will supply your M:F with a #db{} and then remaining args - -get_db_info(DbName) -> - with_db(DbName, {couch_db, get_db_info, []}). - update_docs(DbName, Docs, Options) -> with_db(DbName, {couch_db, update_docs, [Docs, Options]}). -- cgit v1.2.3 From f0161d3167265b6b4e1aaf5755799417c451b415 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Fri, 28 May 2010 15:25:15 -0400 Subject: split fabric_db into dedicated modules --- ebin/fabric.app | 6 +- src/fabric.erl | 8 +- src/fabric_all_databases.erl | 38 ++++++++ src/fabric_create_db.erl | 66 +++++++++++++ src/fabric_db.erl | 217 ------------------------------------------- src/fabric_delete_db.erl | 58 ++++++++++++ src/fabric_get_db_info.erl | 93 +++++++++++++++++++ 7 files changed, 263 insertions(+), 223 deletions(-) create mode 100644 src/fabric_all_databases.erl create mode 100644 src/fabric_create_db.erl delete mode 100644 src/fabric_db.erl create mode 100644 src/fabric_delete_db.erl create mode 100644 src/fabric_get_db_info.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index 363655b2..cf372125 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -5,8 +5,10 @@ {vsn, "0.1.0"}, {modules, [ fabric, - fabric_db, - fabric_doc, + fabric_all_databases, + fabric_create_db, + fabric_delete_db, + fabric_get_db_info, fabric_missing_revs, fabric_open_doc, fabric_open_revs, diff --git a/src/fabric.erl b/src/fabric.erl index 399b76d6..9fdea34c 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -16,16 +16,16 @@ db_path(RawUri, Customer) -> Path. all_databases(Customer) -> - fabric_db:all_databases(Customer). + fabric_all_databases:all_databases(Customer). get_db_info(DbName, Customer) -> - fabric_db:get_db_info(dbname(DbName), Customer). + fabric_get_db_info:get_db_info(dbname(DbName), Customer). create_db(DbName, Options) -> - fabric_db:create_db(dbname(DbName), Options). + fabric_create_db:create_db(dbname(DbName), Options). delete_db(DbName, Options) -> - fabric_db:delete_db(dbname(DbName), Options). + fabric_delete_db:delete_db(dbname(DbName), Options). diff --git a/src/fabric_all_databases.erl b/src/fabric_all_databases.erl new file mode 100644 index 00000000..1b1d4d00 --- /dev/null +++ b/src/fabric_all_databases.erl @@ -0,0 +1,38 @@ +-module(fabric_all_databases). +-author(brad@cloudant.com). + +-export([all_databases/1]). + +-include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). + + +%% @doc gets all databases in the cluster. +-spec all_databases(binary() | []) -> [binary()]. +all_databases([]) -> + Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> + new_acc(DbName, AccIn) + end, [], partitions), + {ok, Dbs}; +all_databases(Customer) -> + ?debugFmt("~nCustomer: ~p~n", [Customer]), + Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> + DbNameStr = ?b2l(DbName), + case string:str(DbNameStr, Customer) of + 1 -> + new_acc(DbNameStr, AccIn); + _ -> AccIn + end + end, [], dbs_cache), + {ok, Dbs}. + + +%% ===================== +%% internal +%% ===================== + +new_acc(DbName, Acc) -> + case lists:member(DbName, Acc) of + true -> Acc; + _ ->[DbName | Acc] + end. diff --git a/src/fabric_create_db.erl b/src/fabric_create_db.erl new file mode 100644 index 00000000..b0ff39ce --- /dev/null +++ b/src/fabric_create_db.erl @@ -0,0 +1,66 @@ +-module(fabric_create_db). +-author(brad@cloudant.com). + +-export([create_db/2]). + +-include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). + + +%% @doc Create a new database, and all its partition files across the cluster +%% Options is proplist with user_ctx, n, q +-spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. +create_db(DbName, Options) -> + Fullmap = partitions:fullmap(DbName, Options), + {ok, FullNodes} = mem3:fullnodes(), + RefPartMap = send_create_calls(Fullmap, Options), + Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || + {_,#shard{range=[Beg,_]}} <- RefPartMap])}, + Result = case fabric_util:receive_loop( + RefPartMap, 1, fun handle_create_msg/3, Acc0) of + {ok, _Results} -> ok; + Error -> Error + end, + % always install partition map, even w/ errors, so delete is possible + partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), + Result. + +%% +%% internal +%% + +%% @doc create the partitions on all appropriate nodes (rexi calls) +-spec send_create_calls(fullmap(), list()) -> [{reference(), part()}]. +send_create_calls(Fullmap, Options) -> + lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> + Ref = rexi:async_server_call({couch_server, Node}, + {create, ShardName, Options}), + {Ref, Part} + end, Fullmap). + +%% @doc handle create messages from shards +handle_create_msg(_, file_exists, _) -> + {error, file_exists}; +handle_create_msg(_, {rexi_EXIT, _Reason}, {Complete, N, Parts}) -> + {ok, {Complete, N-1, Parts}}; +handle_create_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> + if + Complete -> {stop, ok}; + true -> {error, create_db_fubar} + end; +handle_create_msg(_, _, {true, 1, _Acc}) -> + {stop, ok}; +handle_create_msg({_, #shard{range=[Beg,_]}}, {ok, _}, {false, 1, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + case is_complete(PartResults) of + true -> {stop, ok}; + false -> {error, create_db_fubar} + end; +handle_create_msg(_RefPart, {ok, _}, {true, N, Parts}) -> + {ok, {true, N-1, Parts}}; +handle_create_msg({_Ref, #shard{range=[Beg,_]}}, {ok, _}, {false, Rem, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + {ok, {is_complete(PartResults), Rem-1, PartResults}}. + +is_complete(List) -> + lists:all(fun({_,Bool}) -> Bool end, List). diff --git a/src/fabric_db.erl b/src/fabric_db.erl deleted file mode 100644 index 088faa7b..00000000 --- a/src/fabric_db.erl +++ /dev/null @@ -1,217 +0,0 @@ --module(fabric_db). --author('Brad Anderson '). --author('Adam Kocoloski '). - --export([all_databases/1, get_db_info/2, create_db/2, delete_db/2]). - --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - -%% @doc gets all databases in the cluster. --spec all_databases(binary() | []) -> [binary()]. -all_databases([]) -> - Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> - new_acc(DbName, AccIn) - end, [], partitions), - {ok, Dbs}; -all_databases(Customer) -> - ?debugFmt("~nCustomer: ~p~n", [Customer]), - Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> - DbNameStr = ?b2l(DbName), - case string:str(DbNameStr, Customer) of - 1 -> - new_acc(DbNameStr, AccIn); - _ -> AccIn - end - end, [], dbs_cache), - {ok, Dbs}. - -%% @doc get database information tuple -get_db_info(DbName, Customer) -> - Name = cloudant_db_name(Customer, DbName), - Shards = partitions:all_parts(DbName), - Workers = fabric_util:submit_jobs(Shards, get_db_info, []), - Acc0 = {false, length(Workers), lists:usort([ {Beg, nil} || - #shard{range=[Beg,_]} <- Workers])}, - case fabric_util:recv(Workers, #shard.ref, fun handle_info_msg/3, Acc0) of - {ok, ShardInfos} -> - {ok, process_infos(ShardInfos, [{db_name, Name}])}; - Error -> Error - end. - -%% @doc Create a new database, and all its partition files across the cluster -%% Options is proplist with user_ctx, n, q --spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. -create_db(DbName, Options) -> - Fullmap = partitions:fullmap(DbName, Options), - {ok, FullNodes} = mem3:fullnodes(), - RefPartMap = send_create_calls(Fullmap, Options), - Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || - {_,#shard{range=[Beg,_]}} <- RefPartMap])}, - Result = case fabric_util:receive_loop( - RefPartMap, 1, fun handle_create_msg/3, Acc0) of - {ok, _Results} -> ok; - Error -> Error - end, - % always install partition map, even w/ errors, so delete is possible - partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), - Result. - -%% @doc Delete a database, and all its partition files across the cluster -%% Options is proplist with user_ctx, n, q --spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. -delete_db(DbName, Options) -> - Fullmap = partitions:all_parts(DbName), - RefPartMap = send_delete_calls(Fullmap, Options), - Acc0 = {true, length(RefPartMap)}, - case fabric_util:receive_loop( - RefPartMap, 1, fun handle_delete_msg/3, Acc0) of - {ok, _Results} -> - delete_fullmap(DbName), - ok; - Error -> Error - end. - -%% ===================== -%% internal -%% ===================== - -new_acc(DbName, Acc) -> - case lists:member(DbName, Acc) of - true -> Acc; - _ ->[DbName | Acc] - end. - -handle_info_msg(_, _, {true, _, Infos0}) -> - {stop, Infos0}; -handle_info_msg(_, _, {false, 1, Infos0}) -> - MissingShards = lists:reverse(lists:foldl(fun - ({S,nil}, Acc) -> [S|Acc]; - (_, Acc) -> Acc - end, [], Infos0)), - ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), - {error, get_db_info}; -handle_info_msg(#shard{range=[Beg,_]}, {ok, Info}, {false, N, Infos0}) -> - case couch_util:get_value(Beg, Infos0) of - nil -> - Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), - case is_complete(info, Infos) of - true -> {ok, {true, N-1, Infos}}; - false -> {ok, {false, N-1, Infos}} - end; - _ -> - {ok, {false, N-1, Infos0}} - end; -handle_info_msg(_, _Other, {Complete, N, Infos0}) -> - {ok, {Complete, N-1, Infos0}}. - -is_complete(info, List) -> - not lists:any(fun({_,Info}) -> Info =:= nil end, List); -is_complete(create, List) -> - lists:all(fun({_,Bool}) -> Bool end, List). - -cloudant_db_name(Customer, FullName) -> - case Customer of - "" -> FullName; - Name -> re:replace(FullName, [Name,"/"], "", [{return, binary}]) - end. - -%% Loop through Tasks on the flattened Infos and get the aggregated result -process_infos(Infos, Initial) -> - Tasks = [ - {doc_count, fun sum/2, 0}, - {doc_del_count, fun sum/2, 0}, - {update_seq, fun max/2, 1}, - {purge_seq, fun sum/2, 0}, - {compact_running, fun bool/2, 0}, - {disk_size, fun sum/2, 0}, - {instance_start_time, fun(_, _) -> <<"0">> end, 0}, - {disk_format_version, fun max/2, 0}], - - Infos1 = lists:flatten(Infos), - - Result = lists:map(fun({Type, Fun, Default}) -> - {Type, process_info(Type, Fun, Default, Infos1)} - end, Tasks), - lists:flatten([Initial, Result]). - - process_info(Type, Fun, Default, List) -> - lists:foldl(fun(V, AccIn) -> Fun(V, AccIn) end, Default, - proplists:get_all_values(Type, List)). - -sum(New, Existing) -> - New + Existing. - -bool(New, Existing) -> - New andalso Existing. - -max(New, Existing) -> - case New > Existing of - true -> New; - false -> Existing - end. - - -%% @doc create the partitions on all appropriate nodes (rexi calls) --spec send_create_calls(fullmap(), list()) -> [{reference(), part()}]. -send_create_calls(Fullmap, Options) -> - lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> - Ref = rexi:async_server_call({couch_server, Node}, - {create, ShardName, Options}), - {Ref, Part} - end, Fullmap). - -handle_create_msg(_, file_exists, _) -> - {error, file_exists}; -handle_create_msg(_, {rexi_EXIT, _Reason}, {Complete, N, Parts}) -> - {ok, {Complete, N-1, Parts}}; -handle_create_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> - if - Complete -> {stop, ok}; - true -> {error, create_db_fubar} - end; -handle_create_msg(_, _, {true, 1, _Acc}) -> - {stop, ok}; -handle_create_msg({_, #shard{range=[Beg,_]}}, {ok, _}, {false, 1, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - case is_complete(create, PartResults) of - true -> {stop, ok}; - false -> {error, create_db_fubar} - end; -handle_create_msg(_RefPart, {ok, _}, {true, N, Parts}) -> - {ok, {true, N-1, Parts}}; -handle_create_msg({_Ref, #shard{range=[Beg,_]}}, {ok, _}, {false, Rem, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - {ok, {is_complete(create, PartResults), Rem-1, PartResults}}. - - -%% @doc delete the partitions on all appropriate nodes (rexi calls) --spec send_delete_calls(fullmap(), list()) -> [{reference(), part()}]. -send_delete_calls(Parts, Options) -> - lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> - Ref = rexi:async_server_call({couch_server, Node}, - {delete, ShardName, Options}), - {Ref, Part} - end, Parts). - -handle_delete_msg(_, not_found, {NotFound, N}) -> - {ok, {NotFound, N-1}}; -handle_delete_msg(_, {rexi_EXIT, _Reason}, {NotFound, N}) -> - {ok, {NotFound, N-1}}; -handle_delete_msg(_, {rexi_DOWN, _, _, _}, _Acc) -> - {error, delete_db_fubar}; -handle_delete_msg(_, _, {NotFound, 1}) -> - if - NotFound -> {stop, not_found}; - true -> {stop, ok} - end; -handle_delete_msg(_, ok, {_NotFound, N}) -> - {ok, {false, N-1}}. - -delete_fullmap(DbName) -> - case couch_db:open(<<"dbs">>, []) of - {ok, Db} -> - {ok, Doc} = couch_api:open_doc(Db, DbName, nil, []), - couch_api:update_doc(Db, Doc#doc{deleted=true}); - Error -> Error - end. diff --git a/src/fabric_delete_db.erl b/src/fabric_delete_db.erl new file mode 100644 index 00000000..42faf763 --- /dev/null +++ b/src/fabric_delete_db.erl @@ -0,0 +1,58 @@ +-module(fabric_delete_db). +-author(brad@cloudant.com). + +-export([delete_db/2]). + +-include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). + + +%% @doc Delete a database, and all its partition files across the cluster +%% Options is proplist with user_ctx, n, q +-spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. +delete_db(DbName, Options) -> + Fullmap = partitions:all_parts(DbName), + RefPartMap = send_delete_calls(Fullmap, Options), + Acc0 = {true, length(RefPartMap)}, + case fabric_util:receive_loop( + RefPartMap, 1, fun handle_delete_msg/3, Acc0) of + {ok, _Results} -> + delete_fullmap(DbName), + ok; + Error -> Error + end. + +%% +%% internal +%% + +%% @doc delete the partitions on all appropriate nodes (rexi calls) +-spec send_delete_calls(fullmap(), list()) -> [{reference(), part()}]. +send_delete_calls(Parts, Options) -> + lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> + Ref = rexi:async_server_call({couch_server, Node}, + {delete, ShardName, Options}), + {Ref, Part} + end, Parts). + +handle_delete_msg(_, not_found, {NotFound, N}) -> + {ok, {NotFound, N-1}}; +handle_delete_msg(_, {rexi_EXIT, _Reason}, {NotFound, N}) -> + {ok, {NotFound, N-1}}; +handle_delete_msg(_, {rexi_DOWN, _, _, _}, _Acc) -> + {error, delete_db_fubar}; +handle_delete_msg(_, _, {NotFound, 1}) -> + if + NotFound -> {stop, not_found}; + true -> {stop, ok} + end; +handle_delete_msg(_, ok, {_NotFound, N}) -> + {ok, {false, N-1}}. + +delete_fullmap(DbName) -> + case couch_db:open(<<"dbs">>, []) of + {ok, Db} -> + {ok, Doc} = couch_api:open_doc(Db, DbName, nil, []), + couch_api:update_doc(Db, Doc#doc{deleted=true}); + Error -> Error + end. diff --git a/src/fabric_get_db_info.erl b/src/fabric_get_db_info.erl new file mode 100644 index 00000000..cccfde6a --- /dev/null +++ b/src/fabric_get_db_info.erl @@ -0,0 +1,93 @@ +-module(fabric_get_db_info). +-author(brad@cloudant.com). + +-export([get_db_info/2]). + +-include("../../couch/src/couch_db.hrl"). +-include("../../dynomite/include/membership.hrl"). + + +%% @doc get database information tuple +get_db_info(DbName, Customer) -> + Name = cloudant_db_name(Customer, DbName), + Shards = partitions:all_parts(DbName), + Workers = fabric_util:submit_jobs(Shards, get_db_info, []), + Acc0 = {false, length(Workers), lists:usort([ {Beg, nil} || + #shard{range=[Beg,_]} <- Workers])}, + case fabric_util:recv(Workers, #shard.ref, fun handle_info_msg/3, Acc0) of + {ok, ShardInfos} -> + {ok, process_infos(ShardInfos, [{db_name, Name}])}; + Error -> Error + end. + + +%% ===================== +%% internal +%% ===================== + +handle_info_msg(_, _, {true, _, Infos0}) -> + {stop, Infos0}; +handle_info_msg(_, _, {false, 1, Infos0}) -> + MissingShards = lists:reverse(lists:foldl(fun + ({S,nil}, Acc) -> [S|Acc]; + (_, Acc) -> Acc + end, [], Infos0)), + ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), + {error, get_db_info}; +handle_info_msg(#shard{range=[Beg,_]}, {ok, Info}, {false, N, Infos0}) -> + case couch_util:get_value(Beg, Infos0) of + nil -> + Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), + case is_complete(Infos) of + true -> {ok, {true, N-1, Infos}}; + false -> {ok, {false, N-1, Infos}} + end; + _ -> + {ok, {false, N-1, Infos0}} + end; +handle_info_msg(_, _Other, {Complete, N, Infos0}) -> + {ok, {Complete, N-1, Infos0}}. + +is_complete(List) -> + not lists:any(fun({_,Info}) -> Info =:= nil end, List). + +cloudant_db_name(Customer, FullName) -> + case Customer of + "" -> FullName; + Name -> re:replace(FullName, [Name,"/"], "", [{return, binary}]) + end. + +%% Loop through Tasks on the flattened Infos and get the aggregated result +process_infos(Infos, Initial) -> + Tasks = [ + {doc_count, fun sum/2, 0}, + {doc_del_count, fun sum/2, 0}, + {update_seq, fun max/2, 1}, + {purge_seq, fun sum/2, 0}, + {compact_running, fun bool/2, 0}, + {disk_size, fun sum/2, 0}, + {instance_start_time, fun(_, _) -> <<"0">> end, 0}, + {disk_format_version, fun max/2, 0}], + + Infos1 = lists:flatten(Infos), + + Result = lists:map(fun({Type, Fun, Default}) -> + {Type, process_info(Type, Fun, Default, Infos1)} + end, Tasks), + lists:flatten([Initial, Result]). + + process_info(Type, Fun, Default, List) -> + lists:foldl(fun(V, AccIn) -> Fun(V, AccIn) end, Default, + proplists:get_all_values(Type, List)). + +sum(New, Existing) -> + New + Existing. + +bool(New, Existing) -> + New andalso Existing. + +max(New, Existing) -> + case New > Existing of + true -> New; + false -> Existing + end. -- cgit v1.2.3 From 282cf6325145663f450850350471c927465abd70 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 28 May 2010 15:45:57 -0400 Subject: normalizing includes --- include/fabric.hrl | 1 + src/fabric.erl | 6 ++---- src/fabric_all_databases.erl | 4 +--- src/fabric_create_db.erl | 4 +--- src/fabric_delete_db.erl | 4 +--- src/fabric_get_db_info.erl | 4 +--- src/fabric_missing_revs.erl | 2 ++ src/fabric_open_doc.erl | 2 ++ src/fabric_open_revs.erl | 2 ++ src/fabric_rpc.erl | 1 - src/fabric_update_docs.erl | 2 ++ src/fabric_util.erl | 3 +-- 12 files changed, 16 insertions(+), 19 deletions(-) diff --git a/include/fabric.hrl b/include/fabric.hrl index 5426addc..cbd886e4 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -1,2 +1,3 @@ -include("../../couch/src/couch_db.hrl"). -include("../../dynomite/include/membership.hrl"). +-include_lib("eunit/include/eunit.hrl"). diff --git a/src/fabric.erl b/src/fabric.erl index 9fdea34c..7f8f7e23 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,12 +1,10 @@ -module(fabric). --export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, - db_path/2]). +-export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, db_path/2]). -export([open_doc/3, open_revs/4, get_missing_revs/2]). -export([update_doc/3, update_docs/3]). --include("../../couch/src/couch_db.hrl"). - +-include("fabric.hrl"). % db operations -spec db_path(bstring(), bstring()) -> bstring(). diff --git a/src/fabric_all_databases.erl b/src/fabric_all_databases.erl index 1b1d4d00..4bb5c5a5 100644 --- a/src/fabric_all_databases.erl +++ b/src/fabric_all_databases.erl @@ -3,9 +3,7 @@ -export([all_databases/1]). --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - +-include("fabric.hrl"). %% @doc gets all databases in the cluster. -spec all_databases(binary() | []) -> [binary()]. diff --git a/src/fabric_create_db.erl b/src/fabric_create_db.erl index b0ff39ce..1e9bf256 100644 --- a/src/fabric_create_db.erl +++ b/src/fabric_create_db.erl @@ -3,9 +3,7 @@ -export([create_db/2]). --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - +-include("fabric.hrl"). %% @doc Create a new database, and all its partition files across the cluster %% Options is proplist with user_ctx, n, q diff --git a/src/fabric_delete_db.erl b/src/fabric_delete_db.erl index 42faf763..cbcf0d5d 100644 --- a/src/fabric_delete_db.erl +++ b/src/fabric_delete_db.erl @@ -3,9 +3,7 @@ -export([delete_db/2]). --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - +-include("fabric.hrl"). %% @doc Delete a database, and all its partition files across the cluster %% Options is proplist with user_ctx, n, q diff --git a/src/fabric_get_db_info.erl b/src/fabric_get_db_info.erl index cccfde6a..19e72e92 100644 --- a/src/fabric_get_db_info.erl +++ b/src/fabric_get_db_info.erl @@ -3,9 +3,7 @@ -export([get_db_info/2]). --include("../../couch/src/couch_db.hrl"). --include("../../dynomite/include/membership.hrl"). - +-include("fabric.hrl"). %% @doc get database information tuple get_db_info(DbName, Customer) -> diff --git a/src/fabric_missing_revs.erl b/src/fabric_missing_revs.erl index d329d2aa..ee7ea421 100644 --- a/src/fabric_missing_revs.erl +++ b/src/fabric_missing_revs.erl @@ -1,5 +1,7 @@ -module(fabric_missing_revs). + -export([go/2]). + -include("fabric.hrl"). go(DbName, AllIdsRevs) -> diff --git a/src/fabric_open_doc.erl b/src/fabric_open_doc.erl index 996282d8..3ea7d4ab 100644 --- a/src/fabric_open_doc.erl +++ b/src/fabric_open_doc.erl @@ -1,5 +1,7 @@ -module(fabric_open_doc). + -export([go/3]). + -include("fabric.hrl"). go(DbName, Id, Options) -> diff --git a/src/fabric_open_revs.erl b/src/fabric_open_revs.erl index c5bd586d..6847a2e5 100644 --- a/src/fabric_open_revs.erl +++ b/src/fabric_open_revs.erl @@ -1,5 +1,7 @@ -module(fabric_open_revs). + -export([go/4]). + -include("fabric.hrl"). go(DbName, Id, Revs, Options) -> diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index fa67b13f..2d1f6572 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -4,7 +4,6 @@ -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). -include("fabric.hrl"). --include_lib("eunit/include/eunit.hrl"). %% rpc endpoints %% call to with_db will supply your M:F with a #db{} and then remaining args diff --git a/src/fabric_update_docs.erl b/src/fabric_update_docs.erl index 00c7e9d7..1281e40e 100644 --- a/src/fabric_update_docs.erl +++ b/src/fabric_update_docs.erl @@ -1,5 +1,7 @@ -module(fabric_update_docs). + -export([go/3]). + -include("fabric.hrl"). go(DbName, AllDocs, Options) -> diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 936ba4ee..b6d68e09 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -2,8 +2,7 @@ -export([submit_jobs/3, recv/4, receive_loop/4, receive_loop/6]). --include("../../dynomite/include/membership.hrl"). --include_lib("eunit/include/eunit.hrl"). +-include("fabric.hrl"). submit_jobs(Shards, EndPoint, ExtraArgs) -> lists:map(fun(#shard{node=Node, name=ShardName} = Shard) -> -- cgit v1.2.3 From 2c2a4192531e6af514c21c127065dba3af518561 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Fri, 28 May 2010 16:50:35 -0400 Subject: begin move of dynomite to membership --- include/fabric.hrl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/fabric.hrl b/include/fabric.hrl index cbd886e4..ba28e4a3 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -1,3 +1,11 @@ +-define(FABRIC, true). + +-ifndef(COUCH). -include("../../couch/src/couch_db.hrl"). +-endif. + +-ifndef(MEMBERSHIP). -include("../../dynomite/include/membership.hrl"). +-endif. + -include_lib("eunit/include/eunit.hrl"). -- cgit v1.2.3 From dc28af5e331e13283ec3915e60fa0431673d1845 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Fri, 28 May 2010 16:58:35 -0400 Subject: more move of dynomite to membership, it compiles, but untested --- include/fabric.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fabric.hrl b/include/fabric.hrl index ba28e4a3..43c589e0 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -5,7 +5,7 @@ -endif. -ifndef(MEMBERSHIP). --include("../../dynomite/include/membership.hrl"). +-include("../../membership/include/membership.hrl"). -endif. -include_lib("eunit/include/eunit.hrl"). -- cgit v1.2.3 From 9b7831dbbddc97b4134be2fa3b3ec2d2ebc9462b Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 1 Jun 2010 10:39:47 -0400 Subject: RPC endpoint for all_docs w/o keylist, BugzID 10218 --- src/fabric_rpc.erl | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 93 insertions(+), 5 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 2d1f6572..a8e4585b 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -2,17 +2,52 @@ -export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). +-export([all_docs/2]). -include("fabric.hrl"). +-record (view_acc, { + db, + limit, + include_docs, + offset = nil, + reduce_fun = fun couch_db:enum_docs_reduce_to_count/1, + stop_fun, + group_level = 0 +}). + %% rpc endpoints %% call to with_db will supply your M:F with a #db{} and then remaining args +all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> + {ok, Db} = couch_db:open(DbName, []), + #view_query_args{ + start_key = StartKey, + start_docid = StartDocId, + limit = Limit, + skip = Skip, + include_docs = IncludeDocs, + direction = Dir + } = QueryArgs, + StartId = if is_binary(StartKey) -> StartKey; true -> StartDocId end, + Acc0 = #view_acc{ + db = Db, + include_docs = IncludeDocs, + limit = Limit+Skip, + stop_fun = all_docs_stop_fun(QueryArgs) + }, + {ok, Acc} = couch_db:enum_docs(Db, StartId, Dir, fun view_fold/3, Acc0), + if Acc#view_acc.offset == nil -> + Total = couch_db:get_doc_count(Db), + rexi:sync_reply({total_and_offset, Total, Total}); + true -> ok end, + rexi:reply(complete). + get_db_info(DbName) -> with_db(DbName, {couch_db, get_db_info, []}). get_doc_count(DbName) -> - rexi:reply(case couch_db:open(DbName) of + rexi:reply(case couch_db:open(DbName, []) of {ok, Db} -> {ok, {Count, _DelCount}} = couch_btree:full_reduce(Db#db.id_tree), {ok, Count}; @@ -21,7 +56,7 @@ get_doc_count(DbName) -> end). get_update_seq(DbName) -> - rexi:reply(case couch_db:open(DbName) of + rexi:reply(case couch_db:open(DbName, []) of {ok, #db{update_seq = Seq}} -> {ok, Seq}; Error -> @@ -66,7 +101,60 @@ with_db(DbName, {M,F,A}) -> rexi:reply(Error) end. +view_fold(#full_doc_info{} = FullDocInfo, OffsetReds, Acc) -> + % matches for _all_docs and translates #full_doc_info{} -> KV pair + case couch_doc:to_doc_info(FullDocInfo) of + #doc_info{revs=[#rev_info{deleted=false, rev=Rev}|_]} -> + Id = FullDocInfo#full_doc_info.id, + Value = {[{rev,couch_doc:rev_to_str(Rev)}]}, + view_fold({{Id,Id}, Value}, OffsetReds, Acc); + #doc_info{revs=[#rev_info{deleted=true}|_]} -> + {ok, Acc} + end; +view_fold(KV, OffsetReds, #view_acc{offset=nil} = Acc) -> + % calculates the offset for this shard + #view_acc{db=Db, reduce_fun=Reduce} = Acc, + Offset = Reduce(OffsetReds), + rexi:sync_reply({total_and_offset, couch_db:get_doc_count(Db), Offset}), + view_fold(KV, OffsetReds, Acc#view_acc{offset=Offset}); +view_fold(_KV, _Offset, #view_acc{limit=0} = Acc) -> + % we scanned through limit+skip local rows + {stop, Acc}; +view_fold({{Key,Id}, Value}, _Offset, Acc) -> + % the normal case + #view_acc{ + db = Db, + limit = Limit, + include_docs = IncludeDocs, + stop_fun = PassedEnd + } = Acc, + case PassedEnd(Key, Id) of + true -> + {stop, Acc}; + false -> + RowProps = case IncludeDocs of + true -> + case couch_db:open_doc(Db, Id, []) of + {not_found, missing} -> + [{id, Id}, {key, Key}, {value, Value}, {error, missing}]; + {not_found, deleted} -> + [{id, Id}, {key, Key}, {value, Value}]; + {ok, Doc} -> + JsonDoc = couch_doc:to_json_obj(Doc, []), + [{id, Id}, {key, Key}, {value, Value}, {doc, JsonDoc}] + end; + false -> + [{id, Id}, {key, Key}, {value, Value}] + end, + rexi:sync_reply({row, RowProps}), + {ok, Acc#view_acc{limit=Limit-1}} + end. -%% -%% helper funs -%% +all_docs_stop_fun(#view_query_args{direction=fwd, end_key=EndKey}) -> + fun(ViewKey, _) -> + couch_db_updater:less_docid(EndKey, ViewKey) + end; +all_docs_stop_fun(#view_query_args{direction=rev, end_key=EndKey}) -> + fun(ViewKey, _) -> + couch_db_updater:less_docid(ViewKey, EndKey) + end. -- cgit v1.2.3 From bd976a5a78ba88a25996fc7e94c22e8a8925ccec Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 1 Jun 2010 10:58:17 -0400 Subject: add sync_reply, change msg format to be more like gen_server --- src/fabric_create_db.erl | 12 ++++++------ src/fabric_delete_db.erl | 8 ++++---- src/fabric_get_db_info.erl | 4 ++-- src/fabric_missing_revs.erl | 8 ++++---- src/fabric_open_doc.erl | 6 +++--- src/fabric_open_revs.erl | 6 +++--- src/fabric_update_docs.erl | 6 +++--- src/fabric_util.erl | 14 ++++++++++---- 8 files changed, 35 insertions(+), 29 deletions(-) diff --git a/src/fabric_create_db.erl b/src/fabric_create_db.erl index 1e9bf256..21b093bf 100644 --- a/src/fabric_create_db.erl +++ b/src/fabric_create_db.erl @@ -37,26 +37,26 @@ send_create_calls(Fullmap, Options) -> end, Fullmap). %% @doc handle create messages from shards -handle_create_msg(_, file_exists, _) -> +handle_create_msg(file_exists, _, _) -> {error, file_exists}; -handle_create_msg(_, {rexi_EXIT, _Reason}, {Complete, N, Parts}) -> +handle_create_msg({rexi_EXIT, _Reason}, _, {Complete, N, Parts}) -> {ok, {Complete, N-1, Parts}}; -handle_create_msg(_, {rexi_DOWN, _, _, _}, {Complete, _N, _Parts}) -> +handle_create_msg({rexi_DOWN, _, _, _}, _, {Complete, _N, _Parts}) -> if Complete -> {stop, ok}; true -> {error, create_db_fubar} end; handle_create_msg(_, _, {true, 1, _Acc}) -> {stop, ok}; -handle_create_msg({_, #shard{range=[Beg,_]}}, {ok, _}, {false, 1, PartResults0}) -> +handle_create_msg({ok, _}, {_, #shard{range=[Beg,_]}}, {false, 1, PartResults0}) -> PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), case is_complete(PartResults) of true -> {stop, ok}; false -> {error, create_db_fubar} end; -handle_create_msg(_RefPart, {ok, _}, {true, N, Parts}) -> +handle_create_msg({ok, _}, _RefPart, {true, N, Parts}) -> {ok, {true, N-1, Parts}}; -handle_create_msg({_Ref, #shard{range=[Beg,_]}}, {ok, _}, {false, Rem, PartResults0}) -> +handle_create_msg({ok, _}, {_Ref, #shard{range=[Beg,_]}}, {false, Rem, PartResults0}) -> PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), {ok, {is_complete(PartResults), Rem-1, PartResults}}. diff --git a/src/fabric_delete_db.erl b/src/fabric_delete_db.erl index cbcf0d5d..d21a1a06 100644 --- a/src/fabric_delete_db.erl +++ b/src/fabric_delete_db.erl @@ -33,18 +33,18 @@ send_delete_calls(Parts, Options) -> {Ref, Part} end, Parts). -handle_delete_msg(_, not_found, {NotFound, N}) -> +handle_delete_msg(not_found, _, {NotFound, N}) -> {ok, {NotFound, N-1}}; -handle_delete_msg(_, {rexi_EXIT, _Reason}, {NotFound, N}) -> +handle_delete_msg({rexi_EXIT, _Reason}, _, {NotFound, N}) -> {ok, {NotFound, N-1}}; -handle_delete_msg(_, {rexi_DOWN, _, _, _}, _Acc) -> +handle_delete_msg({rexi_DOWN, _, _, _}, _, _Acc) -> {error, delete_db_fubar}; handle_delete_msg(_, _, {NotFound, 1}) -> if NotFound -> {stop, not_found}; true -> {stop, ok} end; -handle_delete_msg(_, ok, {_NotFound, N}) -> +handle_delete_msg(ok, _, {_NotFound, N}) -> {ok, {false, N-1}}. delete_fullmap(DbName) -> diff --git a/src/fabric_get_db_info.erl b/src/fabric_get_db_info.erl index 19e72e92..9242b569 100644 --- a/src/fabric_get_db_info.erl +++ b/src/fabric_get_db_info.erl @@ -32,7 +32,7 @@ handle_info_msg(_, _, {false, 1, Infos0}) -> end, [], Infos0)), ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), {error, get_db_info}; -handle_info_msg(#shard{range=[Beg,_]}, {ok, Info}, {false, N, Infos0}) -> +handle_info_msg({ok, Info}, #shard{range=[Beg,_]}, {false, N, Infos0}) -> case couch_util:get_value(Beg, Infos0) of nil -> Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), @@ -43,7 +43,7 @@ handle_info_msg(#shard{range=[Beg,_]}, {ok, Info}, {false, N, Infos0}) -> _ -> {ok, {false, N-1, Infos0}} end; -handle_info_msg(_, _Other, {Complete, N, Infos0}) -> +handle_info_msg(_Other, _, {Complete, N, Infos0}) -> {ok, {Complete, N-1, Infos0}}. is_complete(List) -> diff --git a/src/fabric_missing_revs.erl b/src/fabric_missing_revs.erl index ee7ea421..ff756425 100644 --- a/src/fabric_missing_revs.erl +++ b/src/fabric_missing_revs.erl @@ -13,14 +13,14 @@ go(DbName, AllIdsRevs) -> Acc0 = {length(Workers), ResultDict}, fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). -handle_message(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> +handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> skip_message(Acc0); -handle_message(_Worker, {rexi_EXIT, _, _, _}, Acc0) -> +handle_message({rexi_EXIT, _, _, _}, _Worker, Acc0) -> skip_message(Acc0); -handle_message(_Worker, {ok, Results}, {1, D0}) -> +handle_message({ok, Results}, _Worker, {1, D0}) -> D = update_dict(D0, Results), {stop, dict:fold(fun force_reply/3, [], D)}; -handle_message(_Worker, {ok, Results}, {WaitingCount, D0}) -> +handle_message({ok, Results}, _Worker, {WaitingCount, D0}) -> D = update_dict(D0, Results), case dict:fold(fun maybe_reply/3, {stop, []}, D) of continue -> diff --git a/src/fabric_open_doc.erl b/src/fabric_open_doc.erl index 3ea7d4ab..e2ea3023 100644 --- a/src/fabric_open_doc.erl +++ b/src/fabric_open_doc.erl @@ -18,11 +18,11 @@ go(DbName, Id, Options) -> Error end. -handle_message(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> +handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> skip_message(Acc0); -handle_message(_Worker, {rexi_EXIT, _Reason}, Acc0) -> +handle_message({rexi_EXIT, _Reason}, _Worker, Acc0) -> skip_message(Acc0); -handle_message(_Worker, Reply, {WaitingCount, R, Replies}) -> +handle_message(Reply, _Worker, {WaitingCount, R, Replies}) -> case merge_read_reply(make_key(Reply), Reply, Replies) of {_, KeyCount} when KeyCount =:= R -> {stop, Reply}; diff --git a/src/fabric_open_revs.erl b/src/fabric_open_revs.erl index 6847a2e5..cc464203 100644 --- a/src/fabric_open_revs.erl +++ b/src/fabric_open_revs.erl @@ -15,11 +15,11 @@ go(DbName, Id, Revs, Options) -> Else end. -handle_message(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> +handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> skip_message(Acc0); -handle_message(_Worker, {rexi_EXIT, _}, Acc0) -> +handle_message({rexi_EXIT, _}, _Worker, Acc0) -> skip_message(Acc0); -handle_message(_Worker, Reply, {WaitingCount, R, Replies}) -> +handle_message(Reply, _Worker, {WaitingCount, R, Replies}) -> case merge_read_reply(make_key(Reply), Reply, Replies) of {_, KeyCount} when KeyCount =:= R -> {stop, Reply}; diff --git a/src/fabric_update_docs.erl b/src/fabric_update_docs.erl index 1281e40e..7a677e1d 100644 --- a/src/fabric_update_docs.erl +++ b/src/fabric_update_docs.erl @@ -19,11 +19,11 @@ go(DbName, AllDocs, Options) -> Else end. -handle_message(_Worker, {rexi_DOWN, _, _, _}, Acc0) -> +handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> skip_message(Acc0); -handle_message(_Worker, {rexi_EXIT, _}, Acc0) -> +handle_message({rexi_EXIT, _}, _Worker, Acc0) -> skip_message(Acc0); -handle_message(Worker, {ok, Replies}, Acc0) -> +handle_message({ok, Replies}, Worker, Acc0) -> {WaitingCount, DocCount, W, GroupedDocs, DocReplyDict0} = Acc0, Docs = couch_util:get_value(Worker, GroupedDocs), DocReplyDict = append_update_replies(Docs, Replies, DocReplyDict0), diff --git a/src/fabric_util.erl b/src/fabric_util.erl index b6d68e09..dd1aaf0a 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -55,10 +55,16 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> false -> % this was some non-matching message which we will ignore {ok, Acc0}; - RefPart -> - % call the Fun that understands the message - %?debugFmt("~nAcc0: ~p~n", [Acc0]), - Fun(RefPart, Msg, Acc0) + Worker -> + Fun(Msg, Worker, Acc0) + end; + {Ref, From, Msg} -> + io:format("process sync_reply {~p,~p} ~p~n", [Ref, From, Msg]), + case lists:keyfind(Ref, Keypos, RefList) of + false -> + {ok, Acc0}; + Worker -> + Fun(Msg, {Worker, From}, Acc0) end; {rexi_DOWN, _RexiMonPid, ServerPid, Reason} = Msg -> showroom_log:message(alert, "rexi_DOWN ~p ~p", [ServerPid, Reason]), -- cgit v1.2.3 From d0db79a612a0c1760465d009e4e014bacf217a06 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 1 Jun 2010 11:06:43 -0400 Subject: better get_doc_count RPC, BugzID 10242 --- src/fabric_rpc.erl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index a8e4585b..0ac58640 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -47,13 +47,7 @@ get_db_info(DbName) -> with_db(DbName, {couch_db, get_db_info, []}). get_doc_count(DbName) -> - rexi:reply(case couch_db:open(DbName, []) of - {ok, Db} -> - {ok, {Count, _DelCount}} = couch_btree:full_reduce(Db#db.id_tree), - {ok, Count}; - Error -> - Error - end). + with_db(DbName, {couch_db, get_doc_count, []}). get_update_seq(DbName) -> rexi:reply(case couch_db:open(DbName, []) of -- cgit v1.2.3 From 03caa800e099bf5d0425f22f64af76bdd24a8754 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 1 Jun 2010 22:58:57 -0400 Subject: more s/dynomite/membership/ stuff --- ebin/fabric.app | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index cf372125..b4e7c461 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -18,6 +18,6 @@ ]}, {registered, []}, {included_applications, []}, - {applications, [kernel, stdlib, couch, rexi, dynomite]}, + {applications, [kernel, stdlib, couch, rexi, membership]}, {start_phases, []} ]}. -- cgit v1.2.3 From b12bcd244bdf743de8c91808cf03417b1ea9dde2 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 1 Jun 2010 23:03:56 -0400 Subject: all_docs resource w/o keylist, BugzID 10218 --- ebin/fabric.app | 1 + include/fabric.hrl | 16 +++ src/fabric.erl | 5 + src/fabric_all_docs.erl | 264 ++++++++++++++++++++++++++++++++++++++++++++++++ src/fabric_rpc.erl | 17 ++-- src/fabric_util.erl | 4 +- 6 files changed, 296 insertions(+), 11 deletions(-) create mode 100644 src/fabric_all_docs.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index b4e7c461..7b072ca1 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -6,6 +6,7 @@ {modules, [ fabric, fabric_all_databases, + fabric_all_docs, fabric_create_db, fabric_delete_db, fabric_get_db_info, diff --git a/include/fabric.hrl b/include/fabric.hrl index 43c589e0..31e4336c 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -9,3 +9,19 @@ -endif. -include_lib("eunit/include/eunit.hrl"). + +-record(collector, { + query_args, + callback, + counters, + buffer_size, + blocked = [], + total_rows = 0, + offset = 0, + rows = [], + skip, + limit, + user_acc +}). + +-record(view_row, {key, id, value, doc, worker}). diff --git a/src/fabric.erl b/src/fabric.erl index 7f8f7e23..ed75a5ab 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -3,6 +3,7 @@ -export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, db_path/2]). -export([open_doc/3, open_revs/4, get_missing_revs/2]). -export([update_doc/3, update_docs/3]). +-export([all_docs/4]). -include("fabric.hrl"). @@ -45,6 +46,10 @@ update_docs(DbName, Docs, Options) -> fabric_update_docs:go(dbname(DbName), docs(Docs), Options). +all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when + is_function(Callback, 2) -> + fabric_all_docs:go(dbname(DbName), QueryArgs, Callback, Acc0). + %% some simple type validation and transcoding dbname(DbName) when is_list(DbName) -> diff --git a/src/fabric_all_docs.erl b/src/fabric_all_docs.erl new file mode 100644 index 00000000..9304570a --- /dev/null +++ b/src/fabric_all_docs.erl @@ -0,0 +1,264 @@ +-module(fabric_all_docs). + +-export([go/4]). + +-include("fabric.hrl"). + +go(DbName, QueryArgs, Callback, Acc0) -> + Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> + Ref = rexi:cast(Node, {fabric_rpc, all_docs, [Name, QueryArgs]}), + Shard#shard{ref = Ref} + end, partitions:all_parts(DbName)), + BufferSize = couch_config:get("fabric", "map_buffer_size", "2"), + #view_query_args{limit = Limit, skip = Skip} = QueryArgs, + State = #collector{ + query_args = QueryArgs, + callback = Callback, + buffer_size = list_to_integer(BufferSize), + counters = init_counters(Workers), + skip = Skip, + limit = Limit, + user_acc = Acc0 + }, + case fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, + State, infinity, 5000) of + {ok, NewState} -> + {ok, NewState#collector.user_acc}; + Error -> + Error + end. + +handle_message({rexi_DOWN, _, _, _}, nil, State) -> + % TODO see if progress can be made here, possibly by removing all shards + % from that node and checking is_progress_possible + {ok, State}; + +handle_message({rexi_EXIT, _}, Worker, State) -> + #collector{callback=Callback, counters=Counters0, user_acc=Acc} = State, + Counters = remove(Worker, Counters0), + case is_progress_possible(Counters) of + true -> + {ok, State#collector{counters = Counters}}; + false -> + Callback({error, dead_shards}, Acc), + {error, dead_shards} + end; + +handle_message({total_and_offset, Tot, Off}, {Worker, From}, State) -> + #collector{ + callback = Callback, + counters = Counters0, + total_rows = Total0, + offset = Offset0, + user_acc = AccIn + } = State, + case lookup_element(Worker, Counters0) of + undefined -> + % this worker lost the race with other partition copies, terminate + gen_server:reply(From, stop), + {ok, State}; + 0 -> + gen_server:reply(From, ok), + Counters1 = update_counter(Worker, 1, Counters0), + Counters2 = remove_overlapping_shards(Worker, Counters1), + Total = Total0 + Tot, + Offset = Offset0 + Off, + case waiting_on_shards(Counters2) of + true -> + {ok, State#collector{ + counters = Counters2, + total_rows = Total, + offset = Offset + }}; + false -> + FinalOffset = erlang:min(Total, Offset+State#collector.skip), + {Go, Acc} = Callback({total_and_offset, Total, FinalOffset}, AccIn), + {Go, State#collector{ + counters = decrement_all_counters(Counters2), + total_rows = Total, + offset = FinalOffset, + user_acc = Acc + }} + end + end; + +handle_message(#view_row{} = Row, {Worker, From}, State) -> + #collector{query_args = Args, counters = Counters0, rows = Rows0} = State, + Dir = Args#view_query_args.direction, + Rows = merge_row(Dir, Row#view_row{worker=Worker}, Rows0), + Counters1 = update_counter(Worker, 1, Counters0), + State1 = State#collector{rows=Rows, counters=Counters1}, + State2 = maybe_pause_worker(Worker, From, State1), + maybe_send_row(State2); + +handle_message(complete, Worker, State) -> + Counters = update_counter(Worker, 1, State#collector.counters), + maybe_send_row(State#collector{counters = Counters}). + + +maybe_pause_worker(Worker, From, State) -> + #collector{buffer_size = BufferSize, counters = Counters} = State, + case lookup_element(Worker, Counters) of + BufferSize -> + State#collector{blocked = [{Worker,From} | State#collector.blocked]}; + _Count -> + gen_server:reply(From, ok), + State + end. + +maybe_resume_worker(Worker, State) -> + #collector{buffer_size = Buffer, counters = C, blocked = B} = State, + case lookup_element(Worker, C) of + Count when Count < Buffer/2 -> + case couch_util:get_value(Worker, B) of + undefined -> + State; + From -> + gen_server:reply(From, ok), + State#collector{blocked = lists:keydelete(Worker, 1, B)} + end; + _Other -> + State + end. + +maybe_send_row(#collector{limit=0} = State) -> + #collector{user_acc=AccIn, callback=Callback} = State, + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc=Acc}}; +maybe_send_row(State) -> + #collector{ + callback = Callback, + counters = Counters, + skip = Skip, + limit = Limit, + user_acc = AccIn + } = State, + case waiting_on_shards(Counters) of + true -> + {ok, State}; + false -> + case get_next_row(State) of + complete -> + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc=Acc}}; + {_, NewState} when Skip > 0 -> + maybe_send_row(NewState#collector{skip=Skip-1, limit=Limit-1}); + {Row, NewState} -> + case Callback(transform_row(Row), AccIn) of + {stop, Acc} -> + {stop, NewState#collector{user_acc=Acc, limit=Limit-1}}; + {ok, Acc} -> + maybe_send_row(NewState#collector{user_acc=Acc, limit=Limit-1}) + end + end + end. + +get_next_row(#collector{rows = []}) -> + complete; +get_next_row(State) -> + #collector{query_args=Args, rows=[Row|Rest], counters=Counters0} = State, + Worker = Row#view_row.worker, + Counters1 = update_counter(Worker, -1, Counters0), + NewState = maybe_resume_worker(Worker, State#collector{counters=Counters1}), + case stop(Args, Row) of + true -> + complete; + false -> + {Row, NewState#collector{rows = Rest}} + end. + +stop(#view_query_args{direction=fwd, end_key=EndKey}, #view_row{id=Id}) -> + couch_db_updater:less_docid(EndKey, Id); +stop(#view_query_args{direction=rev, end_key=EndKey}, #view_row{id=Id}) -> + couch_db_updater:less_docid(Id, EndKey). + +transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}]}}; +transform_row(#view_row{key=Key, id=Id, value=Value, doc={error,Reason}}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}, {error,Reason}]}}; +transform_row(#view_row{key=Key, id=Id, value=Value, doc=Doc}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}, {doc,Doc}]}}. + +merge_row(fwd, Row, Rows) -> + lists:keymerge(#view_row.id, [Row], Rows); +merge_row(rev, Row, Rows) -> + lists:rkeymerge(#view_row.id, [Row], Rows). + +remove_overlapping_shards(#shard{range=[A,B]} = Shard0, Shards) -> + filter(fun(#shard{range=[X,Y]} = Shard, _Value) -> + if Shard =:= Shard0 -> + % we can't remove ourselves + true; + A < B, X >= A, X < B -> + % lower bound is inside our range + false; + A < B, Y > A, Y =< B -> + % upper bound is inside our range + false; + B < A, X >= A orelse B < A, X < B -> + % target shard wraps the key range, lower bound is inside + false; + B < A, Y > A orelse B < A, Y =< B -> + % target shard wraps the key range, upper bound is inside + false; + true -> + true + end + end, Shards). + +%% @doc looks for a fully covered keyrange in the list of counters +-spec is_progress_possible([{#shard{}, non_neg_integer()}]) -> boolean(). +is_progress_possible(Counters) -> + Ranges = fold(fun(#shard{range=[X,Y]}, _, A) -> [{X,Y}|A] end, [], Counters), + [First | Rest] = lists:ukeysort(1, Ranges), + {Head, Tail} = lists:foldl(fun + (_, {Head, Tail}) when Head =:= Tail -> + % this is the success condition, we can fast-forward + {Head, Tail}; + (_, {foo, bar}) -> + % we've already declared failure + {foo, bar}; + ({X,_}, {Head, Tail}) when Head < Tail, X > Tail -> + % gap in the keyrange, we're dead + {foo, bar}; + ({X,Y}, {Head, Tail}) when Head < Tail, X < Y -> + % the normal condition, adding to the tail + {Head, erlang:max(Tail, Y)}; + ({X,Y}, {Head, Tail}) when Head < Tail, X > Y, Y >= Head -> + % we've wrapped all the way around, trigger success condition + {Head, Head}; + ({X,Y}, {Head, Tail}) when Head < Tail, X > Y -> + % this wraps the keyspace, but there's still a gap. We're dead + % TODO technically, another shard could be a superset of this one, and + % we could still be alive. Pretty unlikely though, and impossible if + % we don't allow shards to wrap around the boundary + {foo, bar} + end, First, Rest), + Head =:= Tail. + +% Instead of ets, let's use an ordered keylist. We'll need to revisit if we +% have >> 100 shards, so a private interface is a good idea. - APK June 2010 + +init_counters(Keys) -> + orddict:from_list([{Key,0} || Key <- Keys]). + +decrement_all_counters(Dict) -> + [{K,V-1} || {K,V} <- Dict]. + +update_counter(Key, Incr, Dict0) -> + orddict:update_counter(Key, Incr, Dict0). + +lookup_element(Key, Dict) -> + couch_util:get_value(Key, Dict). + +waiting_on_shards(Dict) -> + lists:keymember(0, 2, Dict). + +remove(Shard, Dict) -> + orddict:erase(Shard, Dict). + +filter(Fun, Dict) -> + orddict:filter(Fun, Dict). + +fold(Fun, Acc0, Dict) -> + orddict:fold(Fun, Acc0, Dict). diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 0ac58640..1cae5ac5 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -38,7 +38,7 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> }, {ok, Acc} = couch_db:enum_docs(Db, StartId, Dir, fun view_fold/3, Acc0), if Acc#view_acc.offset == nil -> - Total = couch_db:get_doc_count(Db), + {ok, Total} = couch_db:get_doc_count(Db), rexi:sync_reply({total_and_offset, Total, Total}); true -> ok end, rexi:reply(complete). @@ -108,8 +108,9 @@ view_fold(#full_doc_info{} = FullDocInfo, OffsetReds, Acc) -> view_fold(KV, OffsetReds, #view_acc{offset=nil} = Acc) -> % calculates the offset for this shard #view_acc{db=Db, reduce_fun=Reduce} = Acc, + {ok, Total} = couch_db:get_doc_count(Db), Offset = Reduce(OffsetReds), - rexi:sync_reply({total_and_offset, couch_db:get_doc_count(Db), Offset}), + rexi:sync_reply({total_and_offset, Total, Offset}), view_fold(KV, OffsetReds, Acc#view_acc{offset=Offset}); view_fold(_KV, _Offset, #view_acc{limit=0} = Acc) -> % we scanned through limit+skip local rows @@ -126,21 +127,21 @@ view_fold({{Key,Id}, Value}, _Offset, Acc) -> true -> {stop, Acc}; false -> - RowProps = case IncludeDocs of + Row = case IncludeDocs of true -> case couch_db:open_doc(Db, Id, []) of {not_found, missing} -> - [{id, Id}, {key, Key}, {value, Value}, {error, missing}]; + #view_row{key=Key, id=Id, value=Value, doc={error,missing}}; {not_found, deleted} -> - [{id, Id}, {key, Key}, {value, Value}]; + #view_row{key=Key, id=Id, value=Value}; {ok, Doc} -> JsonDoc = couch_doc:to_json_obj(Doc, []), - [{id, Id}, {key, Key}, {value, Value}, {doc, JsonDoc}] + #view_row{key=Key, id=Id, value=Value, doc=JsonDoc} end; false -> - [{id, Id}, {key, Key}, {value, Value}] + #view_row{key=Key, id=Id, value=Value} end, - rexi:sync_reply({row, RowProps}), + rexi:sync_reply(Row), {ok, Acc#view_acc{limit=Limit-1}} end. diff --git a/src/fabric_util.erl b/src/fabric_util.erl index dd1aaf0a..e904b456 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -50,7 +50,6 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> {timeout, TimeoutRef} -> timeout; {Ref, Msg} -> - io:format("process_message ~p ~p~n", [Ref, Msg]), case lists:keyfind(Ref, Keypos, RefList) of false -> % this was some non-matching message which we will ignore @@ -59,7 +58,6 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> Fun(Msg, Worker, Acc0) end; {Ref, From, Msg} -> - io:format("process sync_reply {~p,~p} ~p~n", [Ref, From, Msg]), case lists:keyfind(Ref, Keypos, RefList) of false -> {ok, Acc0}; @@ -68,7 +66,7 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> end; {rexi_DOWN, _RexiMonPid, ServerPid, Reason} = Msg -> showroom_log:message(alert, "rexi_DOWN ~p ~p", [ServerPid, Reason]), - Fun(nil, Msg, Acc0) + Fun(Msg, nil, Acc0) after PerMsgTO -> timeout end. -- cgit v1.2.3 From 0340953f4ff3cc6ee09b786031442f97f79090c3 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 2 Jun 2010 12:07:30 -0400 Subject: _all_docs with keys, closes BugzID 10218 --- ebin/fabric.app | 1 + src/fabric.erl | 18 +++++++++++--- src/fabric_all_docs.erl | 66 ++++++++++++++++++++++++++++++++++++++++++++++++- src/fabric_rpc.erl | 22 +++++++---------- src/fabric_util.erl | 2 ++ 5 files changed, 92 insertions(+), 17 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index 7b072ca1..c96e73e9 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -9,6 +9,7 @@ fabric_all_docs, fabric_create_db, fabric_delete_db, + fabric_doc_count, fabric_get_db_info, fabric_missing_revs, fabric_open_doc, diff --git a/src/fabric.erl b/src/fabric.erl index ed75a5ab..fd26abc3 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,10 +1,19 @@ -module(fabric). --export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, db_path/2]). --export([open_doc/3, open_revs/4, get_missing_revs/2]). --export([update_doc/3, update_docs/3]). +% DBs +-export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, + get_doc_count/1]). + +% Documents +-export([open_doc/3, open_revs/4, get_missing_revs/2, update_doc/3, + update_docs/3]). + +% Views -export([all_docs/4]). +% miscellany +-export([db_path/2]). + -include("fabric.hrl"). % db operations @@ -20,6 +29,9 @@ all_databases(Customer) -> get_db_info(DbName, Customer) -> fabric_get_db_info:get_db_info(dbname(DbName), Customer). +get_doc_count(DbName) -> + fabric_doc_count:go(dbname(DbName)). + create_db(DbName, Options) -> fabric_create_db:create_db(dbname(DbName), Options). diff --git a/src/fabric_all_docs.erl b/src/fabric_all_docs.erl index 9304570a..77d21bcd 100644 --- a/src/fabric_all_docs.erl +++ b/src/fabric_all_docs.erl @@ -1,10 +1,11 @@ -module(fabric_all_docs). -export([go/4]). +-export([open_doc/3]). % exported for spawn -include("fabric.hrl"). -go(DbName, QueryArgs, Callback, Acc0) -> +go(DbName, #view_query_args{keys=nil} = QueryArgs, Callback, Acc0) -> Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> Ref = rexi:cast(Node, {fabric_rpc, all_docs, [Name, QueryArgs]}), Shard#shard{ref = Ref} @@ -26,6 +27,26 @@ go(DbName, QueryArgs, Callback, Acc0) -> {ok, NewState#collector.user_acc}; Error -> Error + end; + +go(DbName, QueryArgs, Callback, Acc0) -> + #view_query_args{ + direction = Dir, + include_docs = IncludeDocs, + limit = Limit0, + skip = Skip0, + keys = Keys + } = QueryArgs, + {_, Ref0} = spawn_monitor(fun() -> exit(fabric:get_doc_count(DbName)) end), + Monitors0 = [spawn_monitor(?MODULE, open_doc, [DbName, Id, IncludeDocs]) || + Id <- Keys], + Monitors = if Dir=:=fwd -> Monitors0; true -> lists:reverse(Monitors0) end, + receive {'DOWN', Ref0, _, _, {ok, TotalRows}} -> + {ok, Acc1} = Callback({total_and_offset, TotalRows, 0}, Acc0), + {ok, Acc2} = doc_receive_loop(Monitors, Skip0, Limit0, Callback, Acc1), + Callback(complete, Acc2) + after 10000 -> + Callback(timeout, Acc0) end. handle_message({rexi_DOWN, _, _, _}, nil, State) -> @@ -172,6 +193,8 @@ stop(#view_query_args{direction=fwd, end_key=EndKey}, #view_row{id=Id}) -> stop(#view_query_args{direction=rev, end_key=EndKey}, #view_row{id=Id}) -> couch_db_updater:less_docid(Id, EndKey). +transform_row(#view_row{key=Key, id=undefined}) -> + {row, {[{key,Key}, {error,not_found}]}}; transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> {row, {[{key,Key}, {id,Id}, {value,Value}]}}; transform_row(#view_row{key=Key, id=Id, value=Value, doc={error,Reason}}) -> @@ -236,6 +259,47 @@ is_progress_possible(Counters) -> end, First, Rest), Head =:= Tail. +doc_receive_loop([], _, _, _, Acc) -> + {ok, Acc}; +doc_receive_loop(_, _, 0, _, Acc) -> + {ok, Acc}; +doc_receive_loop([{Pid,Ref}|Rest], Skip, Limit, Callback, Acc) when Skip > 0 -> + receive {'DOWN', Ref, process, Pid, #view_row{}} -> + doc_receive_loop(Rest, Skip-1, Limit-1, Callback, Acc) + after 10000 -> + timeout + end; +doc_receive_loop([{Pid,Ref}|Rest], 0, Limit, Callback, AccIn) -> + receive {'DOWN', Ref, process, Pid, #view_row{} = Row} -> + case Callback(transform_row(Row), AccIn) of + {ok, Acc} -> + doc_receive_loop(Rest, 0, Limit-1, Callback, Acc); + {stop, Acc} -> + {ok, Acc} + end + after 10000 -> + timeout + end. + +open_doc(DbName, Id, IncludeDocs) -> + Row = case fabric:open_doc(DbName, Id, [deleted]) of + {not_found, missing} -> + Doc = undefined, + #view_row{key=Id}; + {ok, #doc{deleted=true, revs=Revs}} -> + Doc = null, + {RevPos, [RevId|_]} = Revs, + Value = {[{rev,couch_doc:rev_to_str({RevPos, RevId})}, {deleted,true}]}, + #view_row{key=Id, id=Id, value=Value}; + {ok, #doc{revs=Revs} = Doc0} -> + Doc = couch_doc:to_json_obj(Doc0, []), + {RevPos, [RevId|_]} = Revs, + Value = {[{rev,couch_doc:rev_to_str({RevPos, RevId})}]}, + #view_row{key=Id, id=Id, value=Value} + end, + exit(if IncludeDocs -> Row#view_row{doc=Doc}; true -> Row end). + + % Instead of ets, let's use an ordered keylist. We'll need to revisit if we % have >> 100 shards, so a private interface is a good idea. - APK June 2010 diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 1cae5ac5..c6aef20b 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -127,21 +127,17 @@ view_fold({{Key,Id}, Value}, _Offset, Acc) -> true -> {stop, Acc}; false -> - Row = case IncludeDocs of - true -> + Doc = if not IncludeDocs -> undefined; true -> case couch_db:open_doc(Db, Id, []) of - {not_found, missing} -> - #view_row{key=Key, id=Id, value=Value, doc={error,missing}}; - {not_found, deleted} -> - #view_row{key=Key, id=Id, value=Value}; - {ok, Doc} -> - JsonDoc = couch_doc:to_json_obj(Doc, []), - #view_row{key=Key, id=Id, value=Value, doc=JsonDoc} - end; - false -> - #view_row{key=Key, id=Id, value=Value} + {not_found, deleted} -> + null; + {not_found, missing} -> + undefined; + {ok, Doc0} -> + couch_doc:to_json_obj(Doc0, []) + end end, - rexi:sync_reply(Row), + rexi:sync_reply(#view_row{key=Key, id=Id, value=Value, doc=Doc}), {ok, Acc#view_acc{limit=Limit-1}} end. diff --git a/src/fabric_util.erl b/src/fabric_util.erl index e904b456..760e511c 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -26,6 +26,8 @@ receive_loop(Workers, Keypos, Fun, Acc0) -> %% @doc set up the receive loop with an overall timeout -spec receive_loop([any()], integer(), function(), any(), timeout(), timeout()) -> {ok, any()}. +receive_loop(RefPartMap, Keypos, Fun, Acc0, infinity, PerMsgTO) -> + process_mailbox(RefPartMap, Keypos, Fun, Acc0, nil, PerMsgTO); receive_loop(RefPartMap, Keypos, Fun, Acc0, GlobalTimeout, PerMsgTO) -> TimeoutRef = erlang:make_ref(), {ok, TRef} = timer:send_after(GlobalTimeout, {timeout, TimeoutRef}), -- cgit v1.2.3 From ca58314b5605906917d5033bc9e75a6c6857f88d Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 2 Jun 2010 12:09:15 -0400 Subject: forgot to commit new file, BugzID 10242 --- src/fabric_doc_count.erl | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/fabric_doc_count.erl diff --git a/src/fabric_doc_count.erl b/src/fabric_doc_count.erl new file mode 100644 index 00000000..75a7a052 --- /dev/null +++ b/src/fabric_doc_count.erl @@ -0,0 +1,42 @@ +-module(fabric_doc_count). + +-export([go/1]). + +-include("fabric.hrl"). + +go(DbName) -> + Shards = partitions:all_parts(DbName), + Workers = fabric_util:submit_jobs(Shards, get_doc_count, []), + Acc0 = {length(Workers), [{Beg,nil} || #shard{range=[Beg,_]} <- Workers]}, + fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). + +handle_message({ok, Count}, #shard{range=[Beg,_]}, {N, Infos0}) -> + case couch_util:get_value(Beg, Infos0) of + nil -> + Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Count}), + case is_complete(Infos) of + true -> + {stop, lists:sum([C || {_, C} <- Infos])}; + false -> + if N > 1 -> + {ok, {N-1, Infos}}; + true -> + report_error(Infos), + {error, missing_shards} + end + end; + _ -> + {ok, {N-1, Infos0}} + end; +handle_message(_, _, {1, Infos}) -> + report_error(Infos), + {error, missing_shards}; +handle_message(_Other, _, {N, Infos0}) -> + {ok, {N-1, Infos0}}. + +report_error(Infos) -> + MissingShards = [S || {S,nil} <- Infos], + ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]). + +is_complete(List) -> + not lists:keymember(nil, 2, List). -- cgit v1.2.3 From 10da0ab42a1902bd98ff41c85e965814f33ecc86 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 2 Jun 2010 12:18:53 -0400 Subject: correct doc_count error message --- src/fabric_doc_count.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fabric_doc_count.erl b/src/fabric_doc_count.erl index 75a7a052..09de9d48 100644 --- a/src/fabric_doc_count.erl +++ b/src/fabric_doc_count.erl @@ -36,7 +36,7 @@ handle_message(_Other, _, {N, Infos0}) -> report_error(Infos) -> MissingShards = [S || {S,nil} <- Infos], - ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]). + ?LOG_ERROR("doc_count error, missing shards: ~p", [MissingShards]). is_complete(List) -> not lists:keymember(nil, 2, List). -- cgit v1.2.3 From 0c5622473b8b2aa90b69e59293b9b23d97bc9557 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 2 Jun 2010 12:24:34 -0400 Subject: all_databases -> all_dbs, less typing ftw --- src/fabric.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index fd26abc3..5c3e896a 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,7 +1,7 @@ -module(fabric). % DBs --export([all_databases/1, create_db/2, delete_db/2, get_db_info/2, +-export([all_dbs/0, all_dbs/1, create_db/2, delete_db/2, get_db_info/2, get_doc_count/1]). % Documents @@ -23,7 +23,10 @@ db_path(RawUri, Customer) -> {Path, _, _} = mochiweb_util:urlsplit_path(CustomerUri), Path. -all_databases(Customer) -> +all_dbs() -> + fabric_all_databases:all_databases(""). + +all_dbs(Customer) -> fabric_all_databases:all_databases(Customer). get_db_info(DbName, Customer) -> -- cgit v1.2.3 From ac0456e1f59133abd646493090731d04d439069f Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Sat, 5 Jun 2010 18:00:32 -0400 Subject: big module renaming --- ebin/fabric.app | 20 +-- src/fabric.erl | 18 +-- src/fabric_all_docs.erl | 328 ---------------------------------------- src/fabric_create_db.erl | 64 -------- src/fabric_db_create.erl | 64 ++++++++ src/fabric_db_delete.erl | 56 +++++++ src/fabric_db_doc_count.erl | 42 +++++ src/fabric_db_info.erl | 91 +++++++++++ src/fabric_delete_db.erl | 56 ------- src/fabric_doc_count.erl | 42 ----- src/fabric_doc_missing_revs.erl | 63 ++++++++ src/fabric_doc_open.erl | 63 ++++++++ src/fabric_doc_open_revs.erl | 62 ++++++++ src/fabric_doc_update.erl | 94 ++++++++++++ src/fabric_get_db_info.erl | 91 ----------- src/fabric_missing_revs.erl | 63 -------- src/fabric_open_doc.erl | 63 -------- src/fabric_open_revs.erl | 62 -------- src/fabric_update_docs.erl | 94 ------------ src/fabric_view_all_docs.erl | 328 ++++++++++++++++++++++++++++++++++++++++ 20 files changed, 882 insertions(+), 882 deletions(-) delete mode 100644 src/fabric_all_docs.erl delete mode 100644 src/fabric_create_db.erl create mode 100644 src/fabric_db_create.erl create mode 100644 src/fabric_db_delete.erl create mode 100644 src/fabric_db_doc_count.erl create mode 100644 src/fabric_db_info.erl delete mode 100644 src/fabric_delete_db.erl delete mode 100644 src/fabric_doc_count.erl create mode 100644 src/fabric_doc_missing_revs.erl create mode 100644 src/fabric_doc_open.erl create mode 100644 src/fabric_doc_open_revs.erl create mode 100644 src/fabric_doc_update.erl delete mode 100644 src/fabric_get_db_info.erl delete mode 100644 src/fabric_missing_revs.erl delete mode 100644 src/fabric_open_doc.erl delete mode 100644 src/fabric_open_revs.erl delete mode 100644 src/fabric_update_docs.erl create mode 100644 src/fabric_view_all_docs.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index c96e73e9..2e0f8df3 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -6,17 +6,17 @@ {modules, [ fabric, fabric_all_databases, - fabric_all_docs, - fabric_create_db, - fabric_delete_db, - fabric_doc_count, - fabric_get_db_info, - fabric_missing_revs, - fabric_open_doc, - fabric_open_revs, + fabric_db_create, + fabric_db_delete, + fabric_db_doc_count, + fabric_db_info, + fabric_doc_missing_revs, + fabric_doc_open, + fabric_doc_open_revs, + fabric_doc_update, fabric_rpc, - fabric_update_docs, - fabric_util + fabric_util, + fabric_view_all_docs ]}, {registered, []}, {included_applications, []}, diff --git a/src/fabric.erl b/src/fabric.erl index 5c3e896a..b493d55c 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -30,40 +30,40 @@ all_dbs(Customer) -> fabric_all_databases:all_databases(Customer). get_db_info(DbName, Customer) -> - fabric_get_db_info:get_db_info(dbname(DbName), Customer). + fabric_db_info:get_db_info(dbname(DbName), Customer). get_doc_count(DbName) -> - fabric_doc_count:go(dbname(DbName)). + fabric_db_doc_count:go(dbname(DbName)). create_db(DbName, Options) -> - fabric_create_db:create_db(dbname(DbName), Options). + fabric_db_create:create_db(dbname(DbName), Options). delete_db(DbName, Options) -> - fabric_delete_db:delete_db(dbname(DbName), Options). + fabric_db_delete:delete_db(dbname(DbName), Options). open_doc(DbName, Id, Options) -> - fabric_open_doc:go(dbname(DbName), docid(Id), Options). + fabric_doc_open:go(dbname(DbName), docid(Id), Options). open_revs(DbName, Id, Revs, Options) -> - fabric_open_revs:go(dbname(DbName), docid(Id), Revs, Options). + fabric_doc_open_revs:go(dbname(DbName), docid(Id), Revs, Options). get_missing_revs(DbName, IdsRevs) when is_list(IdsRevs) -> Sanitized = [idrevs(IdR) || IdR <- IdsRevs], - fabric_missing_revs:go(dbname(DbName), Sanitized). + fabric_doc_missing_revs:go(dbname(DbName), Sanitized). update_doc(DbName, Doc, Options) -> {ok, [Result]} = update_docs(DbName, [Doc], Options), Result. update_docs(DbName, Docs, Options) -> - fabric_update_docs:go(dbname(DbName), docs(Docs), Options). + fabric_doc_update:go(dbname(DbName), docs(Docs), Options). all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when is_function(Callback, 2) -> - fabric_all_docs:go(dbname(DbName), QueryArgs, Callback, Acc0). + fabric_view_all_docs:go(dbname(DbName), QueryArgs, Callback, Acc0). %% some simple type validation and transcoding diff --git a/src/fabric_all_docs.erl b/src/fabric_all_docs.erl deleted file mode 100644 index 77d21bcd..00000000 --- a/src/fabric_all_docs.erl +++ /dev/null @@ -1,328 +0,0 @@ --module(fabric_all_docs). - --export([go/4]). --export([open_doc/3]). % exported for spawn - --include("fabric.hrl"). - -go(DbName, #view_query_args{keys=nil} = QueryArgs, Callback, Acc0) -> - Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> - Ref = rexi:cast(Node, {fabric_rpc, all_docs, [Name, QueryArgs]}), - Shard#shard{ref = Ref} - end, partitions:all_parts(DbName)), - BufferSize = couch_config:get("fabric", "map_buffer_size", "2"), - #view_query_args{limit = Limit, skip = Skip} = QueryArgs, - State = #collector{ - query_args = QueryArgs, - callback = Callback, - buffer_size = list_to_integer(BufferSize), - counters = init_counters(Workers), - skip = Skip, - limit = Limit, - user_acc = Acc0 - }, - case fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, - State, infinity, 5000) of - {ok, NewState} -> - {ok, NewState#collector.user_acc}; - Error -> - Error - end; - -go(DbName, QueryArgs, Callback, Acc0) -> - #view_query_args{ - direction = Dir, - include_docs = IncludeDocs, - limit = Limit0, - skip = Skip0, - keys = Keys - } = QueryArgs, - {_, Ref0} = spawn_monitor(fun() -> exit(fabric:get_doc_count(DbName)) end), - Monitors0 = [spawn_monitor(?MODULE, open_doc, [DbName, Id, IncludeDocs]) || - Id <- Keys], - Monitors = if Dir=:=fwd -> Monitors0; true -> lists:reverse(Monitors0) end, - receive {'DOWN', Ref0, _, _, {ok, TotalRows}} -> - {ok, Acc1} = Callback({total_and_offset, TotalRows, 0}, Acc0), - {ok, Acc2} = doc_receive_loop(Monitors, Skip0, Limit0, Callback, Acc1), - Callback(complete, Acc2) - after 10000 -> - Callback(timeout, Acc0) - end. - -handle_message({rexi_DOWN, _, _, _}, nil, State) -> - % TODO see if progress can be made here, possibly by removing all shards - % from that node and checking is_progress_possible - {ok, State}; - -handle_message({rexi_EXIT, _}, Worker, State) -> - #collector{callback=Callback, counters=Counters0, user_acc=Acc} = State, - Counters = remove(Worker, Counters0), - case is_progress_possible(Counters) of - true -> - {ok, State#collector{counters = Counters}}; - false -> - Callback({error, dead_shards}, Acc), - {error, dead_shards} - end; - -handle_message({total_and_offset, Tot, Off}, {Worker, From}, State) -> - #collector{ - callback = Callback, - counters = Counters0, - total_rows = Total0, - offset = Offset0, - user_acc = AccIn - } = State, - case lookup_element(Worker, Counters0) of - undefined -> - % this worker lost the race with other partition copies, terminate - gen_server:reply(From, stop), - {ok, State}; - 0 -> - gen_server:reply(From, ok), - Counters1 = update_counter(Worker, 1, Counters0), - Counters2 = remove_overlapping_shards(Worker, Counters1), - Total = Total0 + Tot, - Offset = Offset0 + Off, - case waiting_on_shards(Counters2) of - true -> - {ok, State#collector{ - counters = Counters2, - total_rows = Total, - offset = Offset - }}; - false -> - FinalOffset = erlang:min(Total, Offset+State#collector.skip), - {Go, Acc} = Callback({total_and_offset, Total, FinalOffset}, AccIn), - {Go, State#collector{ - counters = decrement_all_counters(Counters2), - total_rows = Total, - offset = FinalOffset, - user_acc = Acc - }} - end - end; - -handle_message(#view_row{} = Row, {Worker, From}, State) -> - #collector{query_args = Args, counters = Counters0, rows = Rows0} = State, - Dir = Args#view_query_args.direction, - Rows = merge_row(Dir, Row#view_row{worker=Worker}, Rows0), - Counters1 = update_counter(Worker, 1, Counters0), - State1 = State#collector{rows=Rows, counters=Counters1}, - State2 = maybe_pause_worker(Worker, From, State1), - maybe_send_row(State2); - -handle_message(complete, Worker, State) -> - Counters = update_counter(Worker, 1, State#collector.counters), - maybe_send_row(State#collector{counters = Counters}). - - -maybe_pause_worker(Worker, From, State) -> - #collector{buffer_size = BufferSize, counters = Counters} = State, - case lookup_element(Worker, Counters) of - BufferSize -> - State#collector{blocked = [{Worker,From} | State#collector.blocked]}; - _Count -> - gen_server:reply(From, ok), - State - end. - -maybe_resume_worker(Worker, State) -> - #collector{buffer_size = Buffer, counters = C, blocked = B} = State, - case lookup_element(Worker, C) of - Count when Count < Buffer/2 -> - case couch_util:get_value(Worker, B) of - undefined -> - State; - From -> - gen_server:reply(From, ok), - State#collector{blocked = lists:keydelete(Worker, 1, B)} - end; - _Other -> - State - end. - -maybe_send_row(#collector{limit=0} = State) -> - #collector{user_acc=AccIn, callback=Callback} = State, - {_, Acc} = Callback(complete, AccIn), - {stop, State#collector{user_acc=Acc}}; -maybe_send_row(State) -> - #collector{ - callback = Callback, - counters = Counters, - skip = Skip, - limit = Limit, - user_acc = AccIn - } = State, - case waiting_on_shards(Counters) of - true -> - {ok, State}; - false -> - case get_next_row(State) of - complete -> - {_, Acc} = Callback(complete, AccIn), - {stop, State#collector{user_acc=Acc}}; - {_, NewState} when Skip > 0 -> - maybe_send_row(NewState#collector{skip=Skip-1, limit=Limit-1}); - {Row, NewState} -> - case Callback(transform_row(Row), AccIn) of - {stop, Acc} -> - {stop, NewState#collector{user_acc=Acc, limit=Limit-1}}; - {ok, Acc} -> - maybe_send_row(NewState#collector{user_acc=Acc, limit=Limit-1}) - end - end - end. - -get_next_row(#collector{rows = []}) -> - complete; -get_next_row(State) -> - #collector{query_args=Args, rows=[Row|Rest], counters=Counters0} = State, - Worker = Row#view_row.worker, - Counters1 = update_counter(Worker, -1, Counters0), - NewState = maybe_resume_worker(Worker, State#collector{counters=Counters1}), - case stop(Args, Row) of - true -> - complete; - false -> - {Row, NewState#collector{rows = Rest}} - end. - -stop(#view_query_args{direction=fwd, end_key=EndKey}, #view_row{id=Id}) -> - couch_db_updater:less_docid(EndKey, Id); -stop(#view_query_args{direction=rev, end_key=EndKey}, #view_row{id=Id}) -> - couch_db_updater:less_docid(Id, EndKey). - -transform_row(#view_row{key=Key, id=undefined}) -> - {row, {[{key,Key}, {error,not_found}]}}; -transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}]}}; -transform_row(#view_row{key=Key, id=Id, value=Value, doc={error,Reason}}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}, {error,Reason}]}}; -transform_row(#view_row{key=Key, id=Id, value=Value, doc=Doc}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}, {doc,Doc}]}}. - -merge_row(fwd, Row, Rows) -> - lists:keymerge(#view_row.id, [Row], Rows); -merge_row(rev, Row, Rows) -> - lists:rkeymerge(#view_row.id, [Row], Rows). - -remove_overlapping_shards(#shard{range=[A,B]} = Shard0, Shards) -> - filter(fun(#shard{range=[X,Y]} = Shard, _Value) -> - if Shard =:= Shard0 -> - % we can't remove ourselves - true; - A < B, X >= A, X < B -> - % lower bound is inside our range - false; - A < B, Y > A, Y =< B -> - % upper bound is inside our range - false; - B < A, X >= A orelse B < A, X < B -> - % target shard wraps the key range, lower bound is inside - false; - B < A, Y > A orelse B < A, Y =< B -> - % target shard wraps the key range, upper bound is inside - false; - true -> - true - end - end, Shards). - -%% @doc looks for a fully covered keyrange in the list of counters --spec is_progress_possible([{#shard{}, non_neg_integer()}]) -> boolean(). -is_progress_possible(Counters) -> - Ranges = fold(fun(#shard{range=[X,Y]}, _, A) -> [{X,Y}|A] end, [], Counters), - [First | Rest] = lists:ukeysort(1, Ranges), - {Head, Tail} = lists:foldl(fun - (_, {Head, Tail}) when Head =:= Tail -> - % this is the success condition, we can fast-forward - {Head, Tail}; - (_, {foo, bar}) -> - % we've already declared failure - {foo, bar}; - ({X,_}, {Head, Tail}) when Head < Tail, X > Tail -> - % gap in the keyrange, we're dead - {foo, bar}; - ({X,Y}, {Head, Tail}) when Head < Tail, X < Y -> - % the normal condition, adding to the tail - {Head, erlang:max(Tail, Y)}; - ({X,Y}, {Head, Tail}) when Head < Tail, X > Y, Y >= Head -> - % we've wrapped all the way around, trigger success condition - {Head, Head}; - ({X,Y}, {Head, Tail}) when Head < Tail, X > Y -> - % this wraps the keyspace, but there's still a gap. We're dead - % TODO technically, another shard could be a superset of this one, and - % we could still be alive. Pretty unlikely though, and impossible if - % we don't allow shards to wrap around the boundary - {foo, bar} - end, First, Rest), - Head =:= Tail. - -doc_receive_loop([], _, _, _, Acc) -> - {ok, Acc}; -doc_receive_loop(_, _, 0, _, Acc) -> - {ok, Acc}; -doc_receive_loop([{Pid,Ref}|Rest], Skip, Limit, Callback, Acc) when Skip > 0 -> - receive {'DOWN', Ref, process, Pid, #view_row{}} -> - doc_receive_loop(Rest, Skip-1, Limit-1, Callback, Acc) - after 10000 -> - timeout - end; -doc_receive_loop([{Pid,Ref}|Rest], 0, Limit, Callback, AccIn) -> - receive {'DOWN', Ref, process, Pid, #view_row{} = Row} -> - case Callback(transform_row(Row), AccIn) of - {ok, Acc} -> - doc_receive_loop(Rest, 0, Limit-1, Callback, Acc); - {stop, Acc} -> - {ok, Acc} - end - after 10000 -> - timeout - end. - -open_doc(DbName, Id, IncludeDocs) -> - Row = case fabric:open_doc(DbName, Id, [deleted]) of - {not_found, missing} -> - Doc = undefined, - #view_row{key=Id}; - {ok, #doc{deleted=true, revs=Revs}} -> - Doc = null, - {RevPos, [RevId|_]} = Revs, - Value = {[{rev,couch_doc:rev_to_str({RevPos, RevId})}, {deleted,true}]}, - #view_row{key=Id, id=Id, value=Value}; - {ok, #doc{revs=Revs} = Doc0} -> - Doc = couch_doc:to_json_obj(Doc0, []), - {RevPos, [RevId|_]} = Revs, - Value = {[{rev,couch_doc:rev_to_str({RevPos, RevId})}]}, - #view_row{key=Id, id=Id, value=Value} - end, - exit(if IncludeDocs -> Row#view_row{doc=Doc}; true -> Row end). - - -% Instead of ets, let's use an ordered keylist. We'll need to revisit if we -% have >> 100 shards, so a private interface is a good idea. - APK June 2010 - -init_counters(Keys) -> - orddict:from_list([{Key,0} || Key <- Keys]). - -decrement_all_counters(Dict) -> - [{K,V-1} || {K,V} <- Dict]. - -update_counter(Key, Incr, Dict0) -> - orddict:update_counter(Key, Incr, Dict0). - -lookup_element(Key, Dict) -> - couch_util:get_value(Key, Dict). - -waiting_on_shards(Dict) -> - lists:keymember(0, 2, Dict). - -remove(Shard, Dict) -> - orddict:erase(Shard, Dict). - -filter(Fun, Dict) -> - orddict:filter(Fun, Dict). - -fold(Fun, Acc0, Dict) -> - orddict:fold(Fun, Acc0, Dict). diff --git a/src/fabric_create_db.erl b/src/fabric_create_db.erl deleted file mode 100644 index 21b093bf..00000000 --- a/src/fabric_create_db.erl +++ /dev/null @@ -1,64 +0,0 @@ --module(fabric_create_db). --author(brad@cloudant.com). - --export([create_db/2]). - --include("fabric.hrl"). - -%% @doc Create a new database, and all its partition files across the cluster -%% Options is proplist with user_ctx, n, q --spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. -create_db(DbName, Options) -> - Fullmap = partitions:fullmap(DbName, Options), - {ok, FullNodes} = mem3:fullnodes(), - RefPartMap = send_create_calls(Fullmap, Options), - Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || - {_,#shard{range=[Beg,_]}} <- RefPartMap])}, - Result = case fabric_util:receive_loop( - RefPartMap, 1, fun handle_create_msg/3, Acc0) of - {ok, _Results} -> ok; - Error -> Error - end, - % always install partition map, even w/ errors, so delete is possible - partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), - Result. - -%% -%% internal -%% - -%% @doc create the partitions on all appropriate nodes (rexi calls) --spec send_create_calls(fullmap(), list()) -> [{reference(), part()}]. -send_create_calls(Fullmap, Options) -> - lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> - Ref = rexi:async_server_call({couch_server, Node}, - {create, ShardName, Options}), - {Ref, Part} - end, Fullmap). - -%% @doc handle create messages from shards -handle_create_msg(file_exists, _, _) -> - {error, file_exists}; -handle_create_msg({rexi_EXIT, _Reason}, _, {Complete, N, Parts}) -> - {ok, {Complete, N-1, Parts}}; -handle_create_msg({rexi_DOWN, _, _, _}, _, {Complete, _N, _Parts}) -> - if - Complete -> {stop, ok}; - true -> {error, create_db_fubar} - end; -handle_create_msg(_, _, {true, 1, _Acc}) -> - {stop, ok}; -handle_create_msg({ok, _}, {_, #shard{range=[Beg,_]}}, {false, 1, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - case is_complete(PartResults) of - true -> {stop, ok}; - false -> {error, create_db_fubar} - end; -handle_create_msg({ok, _}, _RefPart, {true, N, Parts}) -> - {ok, {true, N-1, Parts}}; -handle_create_msg({ok, _}, {_Ref, #shard{range=[Beg,_]}}, {false, Rem, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - {ok, {is_complete(PartResults), Rem-1, PartResults}}. - -is_complete(List) -> - lists:all(fun({_,Bool}) -> Bool end, List). diff --git a/src/fabric_db_create.erl b/src/fabric_db_create.erl new file mode 100644 index 00000000..4f4e3b20 --- /dev/null +++ b/src/fabric_db_create.erl @@ -0,0 +1,64 @@ +-module(fabric_db_create). +-author(brad@cloudant.com). + +-export([create_db/2]). + +-include("fabric.hrl"). + +%% @doc Create a new database, and all its partition files across the cluster +%% Options is proplist with user_ctx, n, q +-spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. +create_db(DbName, Options) -> + Fullmap = partitions:fullmap(DbName, Options), + {ok, FullNodes} = mem3:fullnodes(), + RefPartMap = send_create_calls(Fullmap, Options), + Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || + {_,#shard{range=[Beg,_]}} <- RefPartMap])}, + Result = case fabric_util:receive_loop( + RefPartMap, 1, fun handle_create_msg/3, Acc0) of + {ok, _Results} -> ok; + Error -> Error + end, + % always install partition map, even w/ errors, so delete is possible + partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), + Result. + +%% +%% internal +%% + +%% @doc create the partitions on all appropriate nodes (rexi calls) +-spec send_create_calls(fullmap(), list()) -> [{reference(), part()}]. +send_create_calls(Fullmap, Options) -> + lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> + Ref = rexi:async_server_call({couch_server, Node}, + {create, ShardName, Options}), + {Ref, Part} + end, Fullmap). + +%% @doc handle create messages from shards +handle_create_msg(file_exists, _, _) -> + {error, file_exists}; +handle_create_msg({rexi_EXIT, _Reason}, _, {Complete, N, Parts}) -> + {ok, {Complete, N-1, Parts}}; +handle_create_msg({rexi_DOWN, _, _, _}, _, {Complete, _N, _Parts}) -> + if + Complete -> {stop, ok}; + true -> {error, create_db_fubar} + end; +handle_create_msg(_, _, {true, 1, _Acc}) -> + {stop, ok}; +handle_create_msg({ok, _}, {_, #shard{range=[Beg,_]}}, {false, 1, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + case is_complete(PartResults) of + true -> {stop, ok}; + false -> {error, create_db_fubar} + end; +handle_create_msg({ok, _}, _RefPart, {true, N, Parts}) -> + {ok, {true, N-1, Parts}}; +handle_create_msg({ok, _}, {_Ref, #shard{range=[Beg,_]}}, {false, Rem, PartResults0}) -> + PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), + {ok, {is_complete(PartResults), Rem-1, PartResults}}. + +is_complete(List) -> + lists:all(fun({_,Bool}) -> Bool end, List). diff --git a/src/fabric_db_delete.erl b/src/fabric_db_delete.erl new file mode 100644 index 00000000..0803400d --- /dev/null +++ b/src/fabric_db_delete.erl @@ -0,0 +1,56 @@ +-module(fabric_db_delete). +-author(brad@cloudant.com). + +-export([delete_db/2]). + +-include("fabric.hrl"). + +%% @doc Delete a database, and all its partition files across the cluster +%% Options is proplist with user_ctx, n, q +-spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. +delete_db(DbName, Options) -> + Fullmap = partitions:all_parts(DbName), + RefPartMap = send_delete_calls(Fullmap, Options), + Acc0 = {true, length(RefPartMap)}, + case fabric_util:receive_loop( + RefPartMap, 1, fun handle_delete_msg/3, Acc0) of + {ok, _Results} -> + delete_fullmap(DbName), + ok; + Error -> Error + end. + +%% +%% internal +%% + +%% @doc delete the partitions on all appropriate nodes (rexi calls) +-spec send_delete_calls(fullmap(), list()) -> [{reference(), part()}]. +send_delete_calls(Parts, Options) -> + lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> + Ref = rexi:async_server_call({couch_server, Node}, + {delete, ShardName, Options}), + {Ref, Part} + end, Parts). + +handle_delete_msg(not_found, _, {NotFound, N}) -> + {ok, {NotFound, N-1}}; +handle_delete_msg({rexi_EXIT, _Reason}, _, {NotFound, N}) -> + {ok, {NotFound, N-1}}; +handle_delete_msg({rexi_DOWN, _, _, _}, _, _Acc) -> + {error, delete_db_fubar}; +handle_delete_msg(_, _, {NotFound, 1}) -> + if + NotFound -> {stop, not_found}; + true -> {stop, ok} + end; +handle_delete_msg(ok, _, {_NotFound, N}) -> + {ok, {false, N-1}}. + +delete_fullmap(DbName) -> + case couch_db:open(<<"dbs">>, []) of + {ok, Db} -> + {ok, Doc} = couch_api:open_doc(Db, DbName, nil, []), + couch_api:update_doc(Db, Doc#doc{deleted=true}); + Error -> Error + end. diff --git a/src/fabric_db_doc_count.erl b/src/fabric_db_doc_count.erl new file mode 100644 index 00000000..4c3a72d5 --- /dev/null +++ b/src/fabric_db_doc_count.erl @@ -0,0 +1,42 @@ +-module(fabric_db_doc_count). + +-export([go/1]). + +-include("fabric.hrl"). + +go(DbName) -> + Shards = partitions:all_parts(DbName), + Workers = fabric_util:submit_jobs(Shards, get_doc_count, []), + Acc0 = {length(Workers), [{Beg,nil} || #shard{range=[Beg,_]} <- Workers]}, + fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). + +handle_message({ok, Count}, #shard{range=[Beg,_]}, {N, Infos0}) -> + case couch_util:get_value(Beg, Infos0) of + nil -> + Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Count}), + case is_complete(Infos) of + true -> + {stop, lists:sum([C || {_, C} <- Infos])}; + false -> + if N > 1 -> + {ok, {N-1, Infos}}; + true -> + report_error(Infos), + {error, missing_shards} + end + end; + _ -> + {ok, {N-1, Infos0}} + end; +handle_message(_, _, {1, Infos}) -> + report_error(Infos), + {error, missing_shards}; +handle_message(_Other, _, {N, Infos0}) -> + {ok, {N-1, Infos0}}. + +report_error(Infos) -> + MissingShards = [S || {S,nil} <- Infos], + ?LOG_ERROR("doc_count error, missing shards: ~p", [MissingShards]). + +is_complete(List) -> + not lists:keymember(nil, 2, List). diff --git a/src/fabric_db_info.erl b/src/fabric_db_info.erl new file mode 100644 index 00000000..e70b335c --- /dev/null +++ b/src/fabric_db_info.erl @@ -0,0 +1,91 @@ +-module(fabric_db_info). +-author(brad@cloudant.com). + +-export([get_db_info/2]). + +-include("fabric.hrl"). + +%% @doc get database information tuple +get_db_info(DbName, Customer) -> + Name = cloudant_db_name(Customer, DbName), + Shards = partitions:all_parts(DbName), + Workers = fabric_util:submit_jobs(Shards, get_db_info, []), + Acc0 = {false, length(Workers), lists:usort([ {Beg, nil} || + #shard{range=[Beg,_]} <- Workers])}, + case fabric_util:recv(Workers, #shard.ref, fun handle_info_msg/3, Acc0) of + {ok, ShardInfos} -> + {ok, process_infos(ShardInfos, [{db_name, Name}])}; + Error -> Error + end. + + +%% ===================== +%% internal +%% ===================== + +handle_info_msg(_, _, {true, _, Infos0}) -> + {stop, Infos0}; +handle_info_msg(_, _, {false, 1, Infos0}) -> + MissingShards = lists:reverse(lists:foldl(fun + ({S,nil}, Acc) -> [S|Acc]; + (_, Acc) -> Acc + end, [], Infos0)), + ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), + {error, get_db_info}; +handle_info_msg({ok, Info}, #shard{range=[Beg,_]}, {false, N, Infos0}) -> + case couch_util:get_value(Beg, Infos0) of + nil -> + Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), + case is_complete(Infos) of + true -> {ok, {true, N-1, Infos}}; + false -> {ok, {false, N-1, Infos}} + end; + _ -> + {ok, {false, N-1, Infos0}} + end; +handle_info_msg(_Other, _, {Complete, N, Infos0}) -> + {ok, {Complete, N-1, Infos0}}. + +is_complete(List) -> + not lists:any(fun({_,Info}) -> Info =:= nil end, List). + +cloudant_db_name(Customer, FullName) -> + case Customer of + "" -> FullName; + Name -> re:replace(FullName, [Name,"/"], "", [{return, binary}]) + end. + +%% Loop through Tasks on the flattened Infos and get the aggregated result +process_infos(Infos, Initial) -> + Tasks = [ + {doc_count, fun sum/2, 0}, + {doc_del_count, fun sum/2, 0}, + {update_seq, fun max/2, 1}, + {purge_seq, fun sum/2, 0}, + {compact_running, fun bool/2, 0}, + {disk_size, fun sum/2, 0}, + {instance_start_time, fun(_, _) -> <<"0">> end, 0}, + {disk_format_version, fun max/2, 0}], + + Infos1 = lists:flatten(Infos), + + Result = lists:map(fun({Type, Fun, Default}) -> + {Type, process_info(Type, Fun, Default, Infos1)} + end, Tasks), + lists:flatten([Initial, Result]). + + process_info(Type, Fun, Default, List) -> + lists:foldl(fun(V, AccIn) -> Fun(V, AccIn) end, Default, + proplists:get_all_values(Type, List)). + +sum(New, Existing) -> + New + Existing. + +bool(New, Existing) -> + New andalso Existing. + +max(New, Existing) -> + case New > Existing of + true -> New; + false -> Existing + end. diff --git a/src/fabric_delete_db.erl b/src/fabric_delete_db.erl deleted file mode 100644 index d21a1a06..00000000 --- a/src/fabric_delete_db.erl +++ /dev/null @@ -1,56 +0,0 @@ --module(fabric_delete_db). --author(brad@cloudant.com). - --export([delete_db/2]). - --include("fabric.hrl"). - -%% @doc Delete a database, and all its partition files across the cluster -%% Options is proplist with user_ctx, n, q --spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. -delete_db(DbName, Options) -> - Fullmap = partitions:all_parts(DbName), - RefPartMap = send_delete_calls(Fullmap, Options), - Acc0 = {true, length(RefPartMap)}, - case fabric_util:receive_loop( - RefPartMap, 1, fun handle_delete_msg/3, Acc0) of - {ok, _Results} -> - delete_fullmap(DbName), - ok; - Error -> Error - end. - -%% -%% internal -%% - -%% @doc delete the partitions on all appropriate nodes (rexi calls) --spec send_delete_calls(fullmap(), list()) -> [{reference(), part()}]. -send_delete_calls(Parts, Options) -> - lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> - Ref = rexi:async_server_call({couch_server, Node}, - {delete, ShardName, Options}), - {Ref, Part} - end, Parts). - -handle_delete_msg(not_found, _, {NotFound, N}) -> - {ok, {NotFound, N-1}}; -handle_delete_msg({rexi_EXIT, _Reason}, _, {NotFound, N}) -> - {ok, {NotFound, N-1}}; -handle_delete_msg({rexi_DOWN, _, _, _}, _, _Acc) -> - {error, delete_db_fubar}; -handle_delete_msg(_, _, {NotFound, 1}) -> - if - NotFound -> {stop, not_found}; - true -> {stop, ok} - end; -handle_delete_msg(ok, _, {_NotFound, N}) -> - {ok, {false, N-1}}. - -delete_fullmap(DbName) -> - case couch_db:open(<<"dbs">>, []) of - {ok, Db} -> - {ok, Doc} = couch_api:open_doc(Db, DbName, nil, []), - couch_api:update_doc(Db, Doc#doc{deleted=true}); - Error -> Error - end. diff --git a/src/fabric_doc_count.erl b/src/fabric_doc_count.erl deleted file mode 100644 index 09de9d48..00000000 --- a/src/fabric_doc_count.erl +++ /dev/null @@ -1,42 +0,0 @@ --module(fabric_doc_count). - --export([go/1]). - --include("fabric.hrl"). - -go(DbName) -> - Shards = partitions:all_parts(DbName), - Workers = fabric_util:submit_jobs(Shards, get_doc_count, []), - Acc0 = {length(Workers), [{Beg,nil} || #shard{range=[Beg,_]} <- Workers]}, - fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). - -handle_message({ok, Count}, #shard{range=[Beg,_]}, {N, Infos0}) -> - case couch_util:get_value(Beg, Infos0) of - nil -> - Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Count}), - case is_complete(Infos) of - true -> - {stop, lists:sum([C || {_, C} <- Infos])}; - false -> - if N > 1 -> - {ok, {N-1, Infos}}; - true -> - report_error(Infos), - {error, missing_shards} - end - end; - _ -> - {ok, {N-1, Infos0}} - end; -handle_message(_, _, {1, Infos}) -> - report_error(Infos), - {error, missing_shards}; -handle_message(_Other, _, {N, Infos0}) -> - {ok, {N-1, Infos0}}. - -report_error(Infos) -> - MissingShards = [S || {S,nil} <- Infos], - ?LOG_ERROR("doc_count error, missing shards: ~p", [MissingShards]). - -is_complete(List) -> - not lists:keymember(nil, 2, List). diff --git a/src/fabric_doc_missing_revs.erl b/src/fabric_doc_missing_revs.erl new file mode 100644 index 00000000..fe6deac6 --- /dev/null +++ b/src/fabric_doc_missing_revs.erl @@ -0,0 +1,63 @@ +-module(fabric_doc_missing_revs). + +-export([go/2]). + +-include("fabric.hrl"). + +go(DbName, AllIdsRevs) -> + Workers = lists:map(fun({#shard{name=Name, node=Node} = Shard, IdsRevs}) -> + Ref = rexi:cast(Node, {fabric_rpc, get_missing_revs, [Name, IdsRevs]}), + Shard#shard{ref=Ref} + end, group_idrevs_by_shard(DbName, AllIdsRevs)), + ResultDict = dict:from_list([{Id, {nil,Revs}} || {Id, Revs} <- AllIdsRevs]), + Acc0 = {length(Workers), ResultDict}, + fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). + +handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> + skip_message(Acc0); +handle_message({rexi_EXIT, _, _, _}, _Worker, Acc0) -> + skip_message(Acc0); +handle_message({ok, Results}, _Worker, {1, D0}) -> + D = update_dict(D0, Results), + {stop, dict:fold(fun force_reply/3, [], D)}; +handle_message({ok, Results}, _Worker, {WaitingCount, D0}) -> + D = update_dict(D0, Results), + case dict:fold(fun maybe_reply/3, {stop, []}, D) of + continue -> + % still haven't heard about some Ids + {ok, {WaitingCount - 1, D}}; + {stop, FinalReply} -> + {stop, FinalReply} + end. + +force_reply(Id, {nil,Revs}, Acc) -> + % never heard about this ID, assume it's missing + [{Id, Revs} | Acc]; +force_reply(_, [], Acc) -> + Acc; +force_reply(Id, Revs, Acc) -> + [{Id, Revs} | Acc]. + +maybe_reply(_, _, continue) -> + continue; +maybe_reply(_, {nil, _}, _) -> + continue; +maybe_reply(_, [], {stop, Acc}) -> + {stop, Acc}; +maybe_reply(Id, Revs, {stop, Acc}) -> + {stop, [{Id, Revs} | Acc]}. + +group_idrevs_by_shard(DbName, IdsRevs) -> + dict:to_list(lists:foldl(fun({Id, Revs}, D0) -> + lists:foldl(fun(Shard, D1) -> + dict:append(Shard, {Id, Revs}, D1) + end, D0, partitions:for_key(DbName,Id)) + end, dict:new(), IdsRevs)). + +update_dict(D0, KVs) -> + lists:foldl(fun({K,V}, D1) -> dict:store(K, V, D1) end, D0, KVs). + +skip_message({1, Dict}) -> + {stop, dict:fold(fun force_reply/3, [], Dict)}; +skip_message({WaitingCount, Dict}) -> + {ok, {WaitingCount-1, Dict}}. diff --git a/src/fabric_doc_open.erl b/src/fabric_doc_open.erl new file mode 100644 index 00000000..6f39f39e --- /dev/null +++ b/src/fabric_doc_open.erl @@ -0,0 +1,63 @@ +-module(fabric_doc_open). + +-export([go/3]). + +-include("fabric.hrl"). + +go(DbName, Id, Options) -> + Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_doc, + [Id, Options]), + SuppressDeletedDoc = not lists:member(deleted, Options), + Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, {ok, #doc{deleted=true}}} when SuppressDeletedDoc -> + {not_found, deleted}; + {ok, Else} -> + Else; + Error -> + Error + end. + +handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> + skip_message(Acc0); +handle_message({rexi_EXIT, _Reason}, _Worker, Acc0) -> + skip_message(Acc0); +handle_message(Reply, _Worker, {WaitingCount, R, Replies}) -> + case merge_read_reply(make_key(Reply), Reply, Replies) of + {_, KeyCount} when KeyCount =:= R -> + {stop, Reply}; + {NewReplies, KeyCount} when KeyCount < R -> + if WaitingCount =:= 1 -> + % last message arrived, but still no quorum + repair_read_quorum_failure(NewReplies); + true -> + {ok, {WaitingCount-1, R, NewReplies}} + end + end. + +skip_message({1, _R, Replies}) -> + repair_read_quorum_failure(Replies); +skip_message({WaitingCount, R, Replies}) -> + {ok, {WaitingCount-1, R, Replies}}. + +merge_read_reply(Key, Reply, Replies) -> + case lists:keyfind(Key, 1, Replies) of + false -> + {[{Key, Reply, 1} | Replies], 1}; + {Key, _, N} -> + {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} + end. + +make_key({ok, #doc{id=Id, revs=Revs}}) -> + {Id, Revs}; +make_key(Else) -> + Else. + +repair_read_quorum_failure(Replies) -> + case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of + [] -> + {stop, {not_found, missing}}; + [Doc|_] -> + % TODO merge docs to find the winner as determined by replication + {stop, {ok, Doc}} + end. \ No newline at end of file diff --git a/src/fabric_doc_open_revs.erl b/src/fabric_doc_open_revs.erl new file mode 100644 index 00000000..2fa91208 --- /dev/null +++ b/src/fabric_doc_open_revs.erl @@ -0,0 +1,62 @@ +-module(fabric_doc_open_revs). + +-export([go/4]). + +-include("fabric.hrl"). + +go(DbName, Id, Revs, Options) -> + Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_revs, + [Id, Revs, Options]), + Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, {ok, Reply}} -> + {ok, Reply}; + Else -> + Else + end. + +handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> + skip_message(Acc0); +handle_message({rexi_EXIT, _}, _Worker, Acc0) -> + skip_message(Acc0); +handle_message(Reply, _Worker, {WaitingCount, R, Replies}) -> + case merge_read_reply(make_key(Reply), Reply, Replies) of + {_, KeyCount} when KeyCount =:= R -> + {stop, Reply}; + {NewReplies, KeyCount} when KeyCount < R -> + if WaitingCount =:= 1 -> + % last message arrived, but still no quorum + repair_read_quorum_failure(NewReplies); + true -> + {ok, {WaitingCount-1, R, NewReplies}} + end + end. + +skip_message({1, _R, Replies}) -> + repair_read_quorum_failure(Replies); +skip_message({WaitingCount, R, Replies}) -> + {ok, {WaitingCount-1, R, Replies}}. + +merge_read_reply(Key, Reply, Replies) -> + case lists:keyfind(Key, 1, Replies) of + false -> + {[{Key, Reply, 1} | Replies], 1}; + {Key, _, N} -> + {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} + end. + +make_key({ok, #doc{id=Id, revs=Revs}}) -> + {Id, Revs}; +make_key(Else) -> + Else. + +repair_read_quorum_failure(Replies) -> + case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of + [] -> + {stop, {not_found, missing}}; + [Doc|_] -> + % TODO merge docs to find the winner as determined by replication + {stop, {ok, Doc}} + end. + + \ No newline at end of file diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl new file mode 100644 index 00000000..555b1897 --- /dev/null +++ b/src/fabric_doc_update.erl @@ -0,0 +1,94 @@ +-module(fabric_doc_update). + +-export([go/3]). + +-include("fabric.hrl"). + +go(DbName, AllDocs, Options) -> + GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> + Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name, Docs, Options]}), + {Shard#shard{ref=Ref}, Docs} + end, group_docs_by_shard(DbName, AllDocs)), + {Workers, _} = lists:unzip(GroupedDocs), + Acc0 = {length(Workers), length(AllDocs), couch_util:get_value(w, Options, 1), + GroupedDocs, dict:new()}, + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, Results} -> + {ok, couch_util:reorder_results(AllDocs, Results)}; + Else -> + Else + end. + +handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> + skip_message(Acc0); +handle_message({rexi_EXIT, _}, _Worker, Acc0) -> + skip_message(Acc0); +handle_message({ok, Replies}, Worker, Acc0) -> + {WaitingCount, DocCount, W, GroupedDocs, DocReplyDict0} = Acc0, + Docs = couch_util:get_value(Worker, GroupedDocs), + DocReplyDict = append_update_replies(Docs, Replies, DocReplyDict0), + case {WaitingCount, dict:size(DocReplyDict)} of + {1, _} -> + % last message has arrived, we need to conclude things + {W, Reply} = dict:fold(fun force_reply/3, {W,[]}, DocReplyDict), + {stop, Reply}; + {_, DocCount} -> + % we've got at least one reply for each document, let's take a look + case dict:fold(fun maybe_reply/3, {stop,W,[]}, DocReplyDict) of + continue -> + {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}}; + {stop, W, FinalReplies} -> + {stop, FinalReplies} + end; + {_, N} when N < DocCount -> + % no point in trying to finalize anything yet + {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}} + end. + +force_reply(Doc, Replies, {W, Acc}) -> + % TODO make a real decision here + case Replies of + [] -> + {W, [{Doc, {error, internal_server_error}} | Acc]}; + [Reply| _] -> + {W, [{Doc, Reply} | Acc]} + end. + +maybe_reply(_, _, continue) -> + % we didn't meet quorum for all docs, so we're fast-forwarding the fold + continue; +maybe_reply(Doc, Replies, {stop, W, Acc}) -> + case update_quorum_met(W, Replies) of + {true, Reply} -> + {stop, W, [{Doc, Reply} | Acc]}; + false -> + continue + end. + +update_quorum_met(W, Replies) -> + % TODO make a real decision here + case length(Replies) >= W of + true -> + {true, hd(Replies)}; + false -> + false + end. + +-spec group_docs_by_shard(binary(), [#doc{}]) -> [{#shard{}, [#doc{}]}]. +group_docs_by_shard(DbName, Docs) -> + dict:to_list(lists:foldl(fun(#doc{id=Id} = Doc, D0) -> + lists:foldl(fun(Shard, D1) -> + dict:append(Shard, Doc, D1) + end, D0, partitions:for_key(DbName,Id)) + end, dict:new(), Docs)). + +append_update_replies([], [], DocReplyDict) -> + DocReplyDict; +append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> + % TODO what if the same document shows up twice in one update_docs call? + append_update_replies(Rest1, Rest2, dict:append(Doc, Reply, Dict0)). + +skip_message(Acc0) -> + % TODO fix this + {ok, Acc0}. + diff --git a/src/fabric_get_db_info.erl b/src/fabric_get_db_info.erl deleted file mode 100644 index 9242b569..00000000 --- a/src/fabric_get_db_info.erl +++ /dev/null @@ -1,91 +0,0 @@ --module(fabric_get_db_info). --author(brad@cloudant.com). - --export([get_db_info/2]). - --include("fabric.hrl"). - -%% @doc get database information tuple -get_db_info(DbName, Customer) -> - Name = cloudant_db_name(Customer, DbName), - Shards = partitions:all_parts(DbName), - Workers = fabric_util:submit_jobs(Shards, get_db_info, []), - Acc0 = {false, length(Workers), lists:usort([ {Beg, nil} || - #shard{range=[Beg,_]} <- Workers])}, - case fabric_util:recv(Workers, #shard.ref, fun handle_info_msg/3, Acc0) of - {ok, ShardInfos} -> - {ok, process_infos(ShardInfos, [{db_name, Name}])}; - Error -> Error - end. - - -%% ===================== -%% internal -%% ===================== - -handle_info_msg(_, _, {true, _, Infos0}) -> - {stop, Infos0}; -handle_info_msg(_, _, {false, 1, Infos0}) -> - MissingShards = lists:reverse(lists:foldl(fun - ({S,nil}, Acc) -> [S|Acc]; - (_, Acc) -> Acc - end, [], Infos0)), - ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), - {error, get_db_info}; -handle_info_msg({ok, Info}, #shard{range=[Beg,_]}, {false, N, Infos0}) -> - case couch_util:get_value(Beg, Infos0) of - nil -> - Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), - case is_complete(Infos) of - true -> {ok, {true, N-1, Infos}}; - false -> {ok, {false, N-1, Infos}} - end; - _ -> - {ok, {false, N-1, Infos0}} - end; -handle_info_msg(_Other, _, {Complete, N, Infos0}) -> - {ok, {Complete, N-1, Infos0}}. - -is_complete(List) -> - not lists:any(fun({_,Info}) -> Info =:= nil end, List). - -cloudant_db_name(Customer, FullName) -> - case Customer of - "" -> FullName; - Name -> re:replace(FullName, [Name,"/"], "", [{return, binary}]) - end. - -%% Loop through Tasks on the flattened Infos and get the aggregated result -process_infos(Infos, Initial) -> - Tasks = [ - {doc_count, fun sum/2, 0}, - {doc_del_count, fun sum/2, 0}, - {update_seq, fun max/2, 1}, - {purge_seq, fun sum/2, 0}, - {compact_running, fun bool/2, 0}, - {disk_size, fun sum/2, 0}, - {instance_start_time, fun(_, _) -> <<"0">> end, 0}, - {disk_format_version, fun max/2, 0}], - - Infos1 = lists:flatten(Infos), - - Result = lists:map(fun({Type, Fun, Default}) -> - {Type, process_info(Type, Fun, Default, Infos1)} - end, Tasks), - lists:flatten([Initial, Result]). - - process_info(Type, Fun, Default, List) -> - lists:foldl(fun(V, AccIn) -> Fun(V, AccIn) end, Default, - proplists:get_all_values(Type, List)). - -sum(New, Existing) -> - New + Existing. - -bool(New, Existing) -> - New andalso Existing. - -max(New, Existing) -> - case New > Existing of - true -> New; - false -> Existing - end. diff --git a/src/fabric_missing_revs.erl b/src/fabric_missing_revs.erl deleted file mode 100644 index ff756425..00000000 --- a/src/fabric_missing_revs.erl +++ /dev/null @@ -1,63 +0,0 @@ --module(fabric_missing_revs). - --export([go/2]). - --include("fabric.hrl"). - -go(DbName, AllIdsRevs) -> - Workers = lists:map(fun({#shard{name=Name, node=Node} = Shard, IdsRevs}) -> - Ref = rexi:cast(Node, {fabric_rpc, get_missing_revs, [Name, IdsRevs]}), - Shard#shard{ref=Ref} - end, group_idrevs_by_shard(DbName, AllIdsRevs)), - ResultDict = dict:from_list([{Id, {nil,Revs}} || {Id, Revs} <- AllIdsRevs]), - Acc0 = {length(Workers), ResultDict}, - fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). - -handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> - skip_message(Acc0); -handle_message({rexi_EXIT, _, _, _}, _Worker, Acc0) -> - skip_message(Acc0); -handle_message({ok, Results}, _Worker, {1, D0}) -> - D = update_dict(D0, Results), - {stop, dict:fold(fun force_reply/3, [], D)}; -handle_message({ok, Results}, _Worker, {WaitingCount, D0}) -> - D = update_dict(D0, Results), - case dict:fold(fun maybe_reply/3, {stop, []}, D) of - continue -> - % still haven't heard about some Ids - {ok, {WaitingCount - 1, D}}; - {stop, FinalReply} -> - {stop, FinalReply} - end. - -force_reply(Id, {nil,Revs}, Acc) -> - % never heard about this ID, assume it's missing - [{Id, Revs} | Acc]; -force_reply(_, [], Acc) -> - Acc; -force_reply(Id, Revs, Acc) -> - [{Id, Revs} | Acc]. - -maybe_reply(_, _, continue) -> - continue; -maybe_reply(_, {nil, _}, _) -> - continue; -maybe_reply(_, [], {stop, Acc}) -> - {stop, Acc}; -maybe_reply(Id, Revs, {stop, Acc}) -> - {stop, [{Id, Revs} | Acc]}. - -group_idrevs_by_shard(DbName, IdsRevs) -> - dict:to_list(lists:foldl(fun({Id, Revs}, D0) -> - lists:foldl(fun(Shard, D1) -> - dict:append(Shard, {Id, Revs}, D1) - end, D0, partitions:for_key(DbName,Id)) - end, dict:new(), IdsRevs)). - -update_dict(D0, KVs) -> - lists:foldl(fun({K,V}, D1) -> dict:store(K, V, D1) end, D0, KVs). - -skip_message({1, Dict}) -> - {stop, dict:fold(fun force_reply/3, [], Dict)}; -skip_message({WaitingCount, Dict}) -> - {ok, {WaitingCount-1, Dict}}. diff --git a/src/fabric_open_doc.erl b/src/fabric_open_doc.erl deleted file mode 100644 index e2ea3023..00000000 --- a/src/fabric_open_doc.erl +++ /dev/null @@ -1,63 +0,0 @@ --module(fabric_open_doc). - --export([go/3]). - --include("fabric.hrl"). - -go(DbName, Id, Options) -> - Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_doc, - [Id, Options]), - SuppressDeletedDoc = not lists:member(deleted, Options), - Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, - case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of - {ok, {ok, #doc{deleted=true}}} when SuppressDeletedDoc -> - {not_found, deleted}; - {ok, Else} -> - Else; - Error -> - Error - end. - -handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> - skip_message(Acc0); -handle_message({rexi_EXIT, _Reason}, _Worker, Acc0) -> - skip_message(Acc0); -handle_message(Reply, _Worker, {WaitingCount, R, Replies}) -> - case merge_read_reply(make_key(Reply), Reply, Replies) of - {_, KeyCount} when KeyCount =:= R -> - {stop, Reply}; - {NewReplies, KeyCount} when KeyCount < R -> - if WaitingCount =:= 1 -> - % last message arrived, but still no quorum - repair_read_quorum_failure(NewReplies); - true -> - {ok, {WaitingCount-1, R, NewReplies}} - end - end. - -skip_message({1, _R, Replies}) -> - repair_read_quorum_failure(Replies); -skip_message({WaitingCount, R, Replies}) -> - {ok, {WaitingCount-1, R, Replies}}. - -merge_read_reply(Key, Reply, Replies) -> - case lists:keyfind(Key, 1, Replies) of - false -> - {[{Key, Reply, 1} | Replies], 1}; - {Key, _, N} -> - {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} - end. - -make_key({ok, #doc{id=Id, revs=Revs}}) -> - {Id, Revs}; -make_key(Else) -> - Else. - -repair_read_quorum_failure(Replies) -> - case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of - [] -> - {stop, {not_found, missing}}; - [Doc|_] -> - % TODO merge docs to find the winner as determined by replication - {stop, {ok, Doc}} - end. \ No newline at end of file diff --git a/src/fabric_open_revs.erl b/src/fabric_open_revs.erl deleted file mode 100644 index cc464203..00000000 --- a/src/fabric_open_revs.erl +++ /dev/null @@ -1,62 +0,0 @@ --module(fabric_open_revs). - --export([go/4]). - --include("fabric.hrl"). - -go(DbName, Id, Revs, Options) -> - Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_revs, - [Id, Revs, Options]), - Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, - case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of - {ok, {ok, Reply}} -> - {ok, Reply}; - Else -> - Else - end. - -handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> - skip_message(Acc0); -handle_message({rexi_EXIT, _}, _Worker, Acc0) -> - skip_message(Acc0); -handle_message(Reply, _Worker, {WaitingCount, R, Replies}) -> - case merge_read_reply(make_key(Reply), Reply, Replies) of - {_, KeyCount} when KeyCount =:= R -> - {stop, Reply}; - {NewReplies, KeyCount} when KeyCount < R -> - if WaitingCount =:= 1 -> - % last message arrived, but still no quorum - repair_read_quorum_failure(NewReplies); - true -> - {ok, {WaitingCount-1, R, NewReplies}} - end - end. - -skip_message({1, _R, Replies}) -> - repair_read_quorum_failure(Replies); -skip_message({WaitingCount, R, Replies}) -> - {ok, {WaitingCount-1, R, Replies}}. - -merge_read_reply(Key, Reply, Replies) -> - case lists:keyfind(Key, 1, Replies) of - false -> - {[{Key, Reply, 1} | Replies], 1}; - {Key, _, N} -> - {lists:keyreplace(Key, 1, Replies, {Key, Reply, N+1}), N+1} - end. - -make_key({ok, #doc{id=Id, revs=Revs}}) -> - {Id, Revs}; -make_key(Else) -> - Else. - -repair_read_quorum_failure(Replies) -> - case [Doc || {_Key, {ok, Doc}, _Count} <- Replies] of - [] -> - {stop, {not_found, missing}}; - [Doc|_] -> - % TODO merge docs to find the winner as determined by replication - {stop, {ok, Doc}} - end. - - \ No newline at end of file diff --git a/src/fabric_update_docs.erl b/src/fabric_update_docs.erl deleted file mode 100644 index 7a677e1d..00000000 --- a/src/fabric_update_docs.erl +++ /dev/null @@ -1,94 +0,0 @@ --module(fabric_update_docs). - --export([go/3]). - --include("fabric.hrl"). - -go(DbName, AllDocs, Options) -> - GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> - Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name, Docs, Options]}), - {Shard#shard{ref=Ref}, Docs} - end, group_docs_by_shard(DbName, AllDocs)), - {Workers, _} = lists:unzip(GroupedDocs), - Acc0 = {length(Workers), length(AllDocs), couch_util:get_value(w, Options, 1), - GroupedDocs, dict:new()}, - case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of - {ok, Results} -> - {ok, couch_util:reorder_results(AllDocs, Results)}; - Else -> - Else - end. - -handle_message({rexi_DOWN, _, _, _}, _Worker, Acc0) -> - skip_message(Acc0); -handle_message({rexi_EXIT, _}, _Worker, Acc0) -> - skip_message(Acc0); -handle_message({ok, Replies}, Worker, Acc0) -> - {WaitingCount, DocCount, W, GroupedDocs, DocReplyDict0} = Acc0, - Docs = couch_util:get_value(Worker, GroupedDocs), - DocReplyDict = append_update_replies(Docs, Replies, DocReplyDict0), - case {WaitingCount, dict:size(DocReplyDict)} of - {1, _} -> - % last message has arrived, we need to conclude things - {W, Reply} = dict:fold(fun force_reply/3, {W,[]}, DocReplyDict), - {stop, Reply}; - {_, DocCount} -> - % we've got at least one reply for each document, let's take a look - case dict:fold(fun maybe_reply/3, {stop,W,[]}, DocReplyDict) of - continue -> - {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}}; - {stop, W, FinalReplies} -> - {stop, FinalReplies} - end; - {_, N} when N < DocCount -> - % no point in trying to finalize anything yet - {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}} - end. - -force_reply(Doc, Replies, {W, Acc}) -> - % TODO make a real decision here - case Replies of - [] -> - {W, [{Doc, {error, internal_server_error}} | Acc]}; - [Reply| _] -> - {W, [{Doc, Reply} | Acc]} - end. - -maybe_reply(_, _, continue) -> - % we didn't meet quorum for all docs, so we're fast-forwarding the fold - continue; -maybe_reply(Doc, Replies, {stop, W, Acc}) -> - case update_quorum_met(W, Replies) of - {true, Reply} -> - {stop, W, [{Doc, Reply} | Acc]}; - false -> - continue - end. - -update_quorum_met(W, Replies) -> - % TODO make a real decision here - case length(Replies) >= W of - true -> - {true, hd(Replies)}; - false -> - false - end. - --spec group_docs_by_shard(binary(), [#doc{}]) -> [{#shard{}, [#doc{}]}]. -group_docs_by_shard(DbName, Docs) -> - dict:to_list(lists:foldl(fun(#doc{id=Id} = Doc, D0) -> - lists:foldl(fun(Shard, D1) -> - dict:append(Shard, Doc, D1) - end, D0, partitions:for_key(DbName,Id)) - end, dict:new(), Docs)). - -append_update_replies([], [], DocReplyDict) -> - DocReplyDict; -append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> - % TODO what if the same document shows up twice in one update_docs call? - append_update_replies(Rest1, Rest2, dict:append(Doc, Reply, Dict0)). - -skip_message(Acc0) -> - % TODO fix this - {ok, Acc0}. - diff --git a/src/fabric_view_all_docs.erl b/src/fabric_view_all_docs.erl new file mode 100644 index 00000000..b91f0665 --- /dev/null +++ b/src/fabric_view_all_docs.erl @@ -0,0 +1,328 @@ +-module(fabric_view_all_docs). + +-export([go/4]). +-export([open_doc/3]). % exported for spawn + +-include("fabric.hrl"). + +go(DbName, #view_query_args{keys=nil} = QueryArgs, Callback, Acc0) -> + Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> + Ref = rexi:cast(Node, {fabric_rpc, all_docs, [Name, QueryArgs]}), + Shard#shard{ref = Ref} + end, partitions:all_parts(DbName)), + BufferSize = couch_config:get("fabric", "map_buffer_size", "2"), + #view_query_args{limit = Limit, skip = Skip} = QueryArgs, + State = #collector{ + query_args = QueryArgs, + callback = Callback, + buffer_size = list_to_integer(BufferSize), + counters = init_counters(Workers), + skip = Skip, + limit = Limit, + user_acc = Acc0 + }, + case fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, + State, infinity, 5000) of + {ok, NewState} -> + {ok, NewState#collector.user_acc}; + Error -> + Error + end; + +go(DbName, QueryArgs, Callback, Acc0) -> + #view_query_args{ + direction = Dir, + include_docs = IncludeDocs, + limit = Limit0, + skip = Skip0, + keys = Keys + } = QueryArgs, + {_, Ref0} = spawn_monitor(fun() -> exit(fabric:get_doc_count(DbName)) end), + Monitors0 = [spawn_monitor(?MODULE, open_doc, [DbName, Id, IncludeDocs]) || + Id <- Keys], + Monitors = if Dir=:=fwd -> Monitors0; true -> lists:reverse(Monitors0) end, + receive {'DOWN', Ref0, _, _, {ok, TotalRows}} -> + {ok, Acc1} = Callback({total_and_offset, TotalRows, 0}, Acc0), + {ok, Acc2} = doc_receive_loop(Monitors, Skip0, Limit0, Callback, Acc1), + Callback(complete, Acc2) + after 10000 -> + Callback(timeout, Acc0) + end. + +handle_message({rexi_DOWN, _, _, _}, nil, State) -> + % TODO see if progress can be made here, possibly by removing all shards + % from that node and checking is_progress_possible + {ok, State}; + +handle_message({rexi_EXIT, _}, Worker, State) -> + #collector{callback=Callback, counters=Counters0, user_acc=Acc} = State, + Counters = remove(Worker, Counters0), + case is_progress_possible(Counters) of + true -> + {ok, State#collector{counters = Counters}}; + false -> + Callback({error, dead_shards}, Acc), + {error, dead_shards} + end; + +handle_message({total_and_offset, Tot, Off}, {Worker, From}, State) -> + #collector{ + callback = Callback, + counters = Counters0, + total_rows = Total0, + offset = Offset0, + user_acc = AccIn + } = State, + case lookup_element(Worker, Counters0) of + undefined -> + % this worker lost the race with other partition copies, terminate + gen_server:reply(From, stop), + {ok, State}; + 0 -> + gen_server:reply(From, ok), + Counters1 = update_counter(Worker, 1, Counters0), + Counters2 = remove_overlapping_shards(Worker, Counters1), + Total = Total0 + Tot, + Offset = Offset0 + Off, + case waiting_on_shards(Counters2) of + true -> + {ok, State#collector{ + counters = Counters2, + total_rows = Total, + offset = Offset + }}; + false -> + FinalOffset = erlang:min(Total, Offset+State#collector.skip), + {Go, Acc} = Callback({total_and_offset, Total, FinalOffset}, AccIn), + {Go, State#collector{ + counters = decrement_all_counters(Counters2), + total_rows = Total, + offset = FinalOffset, + user_acc = Acc + }} + end + end; + +handle_message(#view_row{} = Row, {Worker, From}, State) -> + #collector{query_args = Args, counters = Counters0, rows = Rows0} = State, + Dir = Args#view_query_args.direction, + Rows = merge_row(Dir, Row#view_row{worker=Worker}, Rows0), + Counters1 = update_counter(Worker, 1, Counters0), + State1 = State#collector{rows=Rows, counters=Counters1}, + State2 = maybe_pause_worker(Worker, From, State1), + maybe_send_row(State2); + +handle_message(complete, Worker, State) -> + Counters = update_counter(Worker, 1, State#collector.counters), + maybe_send_row(State#collector{counters = Counters}). + + +maybe_pause_worker(Worker, From, State) -> + #collector{buffer_size = BufferSize, counters = Counters} = State, + case lookup_element(Worker, Counters) of + BufferSize -> + State#collector{blocked = [{Worker,From} | State#collector.blocked]}; + _Count -> + gen_server:reply(From, ok), + State + end. + +maybe_resume_worker(Worker, State) -> + #collector{buffer_size = Buffer, counters = C, blocked = B} = State, + case lookup_element(Worker, C) of + Count when Count < Buffer/2 -> + case couch_util:get_value(Worker, B) of + undefined -> + State; + From -> + gen_server:reply(From, ok), + State#collector{blocked = lists:keydelete(Worker, 1, B)} + end; + _Other -> + State + end. + +maybe_send_row(#collector{limit=0} = State) -> + #collector{user_acc=AccIn, callback=Callback} = State, + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc=Acc}}; +maybe_send_row(State) -> + #collector{ + callback = Callback, + counters = Counters, + skip = Skip, + limit = Limit, + user_acc = AccIn + } = State, + case waiting_on_shards(Counters) of + true -> + {ok, State}; + false -> + case get_next_row(State) of + complete -> + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc=Acc}}; + {_, NewState} when Skip > 0 -> + maybe_send_row(NewState#collector{skip=Skip-1, limit=Limit-1}); + {Row, NewState} -> + case Callback(transform_row(Row), AccIn) of + {stop, Acc} -> + {stop, NewState#collector{user_acc=Acc, limit=Limit-1}}; + {ok, Acc} -> + maybe_send_row(NewState#collector{user_acc=Acc, limit=Limit-1}) + end + end + end. + +get_next_row(#collector{rows = []}) -> + complete; +get_next_row(State) -> + #collector{query_args=Args, rows=[Row|Rest], counters=Counters0} = State, + Worker = Row#view_row.worker, + Counters1 = update_counter(Worker, -1, Counters0), + NewState = maybe_resume_worker(Worker, State#collector{counters=Counters1}), + case stop(Args, Row) of + true -> + complete; + false -> + {Row, NewState#collector{rows = Rest}} + end. + +stop(#view_query_args{direction=fwd, end_key=EndKey}, #view_row{id=Id}) -> + couch_db_updater:less_docid(EndKey, Id); +stop(#view_query_args{direction=rev, end_key=EndKey}, #view_row{id=Id}) -> + couch_db_updater:less_docid(Id, EndKey). + +transform_row(#view_row{key=Key, id=undefined}) -> + {row, {[{key,Key}, {error,not_found}]}}; +transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}]}}; +transform_row(#view_row{key=Key, id=Id, value=Value, doc={error,Reason}}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}, {error,Reason}]}}; +transform_row(#view_row{key=Key, id=Id, value=Value, doc=Doc}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}, {doc,Doc}]}}. + +merge_row(fwd, Row, Rows) -> + lists:keymerge(#view_row.id, [Row], Rows); +merge_row(rev, Row, Rows) -> + lists:rkeymerge(#view_row.id, [Row], Rows). + +remove_overlapping_shards(#shard{range=[A,B]} = Shard0, Shards) -> + filter(fun(#shard{range=[X,Y]} = Shard, _Value) -> + if Shard =:= Shard0 -> + % we can't remove ourselves + true; + A < B, X >= A, X < B -> + % lower bound is inside our range + false; + A < B, Y > A, Y =< B -> + % upper bound is inside our range + false; + B < A, X >= A orelse B < A, X < B -> + % target shard wraps the key range, lower bound is inside + false; + B < A, Y > A orelse B < A, Y =< B -> + % target shard wraps the key range, upper bound is inside + false; + true -> + true + end + end, Shards). + +%% @doc looks for a fully covered keyrange in the list of counters +-spec is_progress_possible([{#shard{}, non_neg_integer()}]) -> boolean(). +is_progress_possible(Counters) -> + Ranges = fold(fun(#shard{range=[X,Y]}, _, A) -> [{X,Y}|A] end, [], Counters), + [First | Rest] = lists:ukeysort(1, Ranges), + {Head, Tail} = lists:foldl(fun + (_, {Head, Tail}) when Head =:= Tail -> + % this is the success condition, we can fast-forward + {Head, Tail}; + (_, {foo, bar}) -> + % we've already declared failure + {foo, bar}; + ({X,_}, {Head, Tail}) when Head < Tail, X > Tail -> + % gap in the keyrange, we're dead + {foo, bar}; + ({X,Y}, {Head, Tail}) when Head < Tail, X < Y -> + % the normal condition, adding to the tail + {Head, erlang:max(Tail, Y)}; + ({X,Y}, {Head, Tail}) when Head < Tail, X > Y, Y >= Head -> + % we've wrapped all the way around, trigger success condition + {Head, Head}; + ({X,Y}, {Head, Tail}) when Head < Tail, X > Y -> + % this wraps the keyspace, but there's still a gap. We're dead + % TODO technically, another shard could be a superset of this one, and + % we could still be alive. Pretty unlikely though, and impossible if + % we don't allow shards to wrap around the boundary + {foo, bar} + end, First, Rest), + Head =:= Tail. + +doc_receive_loop([], _, _, _, Acc) -> + {ok, Acc}; +doc_receive_loop(_, _, 0, _, Acc) -> + {ok, Acc}; +doc_receive_loop([{Pid,Ref}|Rest], Skip, Limit, Callback, Acc) when Skip > 0 -> + receive {'DOWN', Ref, process, Pid, #view_row{}} -> + doc_receive_loop(Rest, Skip-1, Limit-1, Callback, Acc) + after 10000 -> + timeout + end; +doc_receive_loop([{Pid,Ref}|Rest], 0, Limit, Callback, AccIn) -> + receive {'DOWN', Ref, process, Pid, #view_row{} = Row} -> + case Callback(transform_row(Row), AccIn) of + {ok, Acc} -> + doc_receive_loop(Rest, 0, Limit-1, Callback, Acc); + {stop, Acc} -> + {ok, Acc} + end + after 10000 -> + timeout + end. + +open_doc(DbName, Id, IncludeDocs) -> + Row = case fabric:open_doc(DbName, Id, [deleted]) of + {not_found, missing} -> + Doc = undefined, + #view_row{key=Id}; + {ok, #doc{deleted=true, revs=Revs}} -> + Doc = null, + {RevPos, [RevId|_]} = Revs, + Value = {[{rev,couch_doc:rev_to_str({RevPos, RevId})}, {deleted,true}]}, + #view_row{key=Id, id=Id, value=Value}; + {ok, #doc{revs=Revs} = Doc0} -> + Doc = couch_doc:to_json_obj(Doc0, []), + {RevPos, [RevId|_]} = Revs, + Value = {[{rev,couch_doc:rev_to_str({RevPos, RevId})}]}, + #view_row{key=Id, id=Id, value=Value} + end, + exit(if IncludeDocs -> Row#view_row{doc=Doc}; true -> Row end). + + +% Instead of ets, let's use an ordered keylist. We'll need to revisit if we +% have >> 100 shards, so a private interface is a good idea. - APK June 2010 + +init_counters(Keys) -> + orddict:from_list([{Key,0} || Key <- Keys]). + +decrement_all_counters(Dict) -> + [{K,V-1} || {K,V} <- Dict]. + +update_counter(Key, Incr, Dict0) -> + orddict:update_counter(Key, Incr, Dict0). + +lookup_element(Key, Dict) -> + couch_util:get_value(Key, Dict). + +waiting_on_shards(Dict) -> + lists:keymember(0, 2, Dict). + +remove(Shard, Dict) -> + orddict:erase(Shard, Dict). + +filter(Fun, Dict) -> + orddict:filter(Fun, Dict). + +fold(Fun, Acc0, Dict) -> + orddict:fold(Fun, Acc0, Dict). -- cgit v1.2.3 From ea6edcec3944e502e3a6aade48d95e62c646f27f Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Sat, 5 Jun 2010 18:26:46 -0400 Subject: if collector tells us to stop, we stop --- src/fabric_rpc.erl | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index c6aef20b..81a8ba48 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -37,11 +37,7 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> stop_fun = all_docs_stop_fun(QueryArgs) }, {ok, Acc} = couch_db:enum_docs(Db, StartId, Dir, fun view_fold/3, Acc0), - if Acc#view_acc.offset == nil -> - {ok, Total} = couch_db:get_doc_count(Db), - rexi:sync_reply({total_and_offset, Total, Total}); - true -> ok end, - rexi:reply(complete). + final_all_docs_response(Db, Acc#view_acc.offset). get_db_info(DbName) -> with_db(DbName, {couch_db, get_db_info, []}). @@ -110,8 +106,12 @@ view_fold(KV, OffsetReds, #view_acc{offset=nil} = Acc) -> #view_acc{db=Db, reduce_fun=Reduce} = Acc, {ok, Total} = couch_db:get_doc_count(Db), Offset = Reduce(OffsetReds), - rexi:sync_reply({total_and_offset, Total, Offset}), - view_fold(KV, OffsetReds, Acc#view_acc{offset=Offset}); + case rexi:sync_reply({total_and_offset, Total, Offset}) of + ok -> + view_fold(KV, OffsetReds, Acc#view_acc{offset=Offset}); + stop -> + exit(normal) + end; view_fold(_KV, _Offset, #view_acc{limit=0} = Acc) -> % we scanned through limit+skip local rows {stop, Acc}; @@ -149,3 +149,11 @@ all_docs_stop_fun(#view_query_args{direction=rev, end_key=EndKey}) -> fun(ViewKey, _) -> couch_db_updater:less_docid(ViewKey, EndKey) end. + +final_all_docs_response(Db, nil) -> + {ok, Total} = couch_db:get_doc_count(Db), + case rexi:sync_reply({total_and_offset, Total, Total}) of ok -> + rexi:reply(complete); + stop -> ok end; +final_all_docs_response(_Db, _Offset) -> + rexi:reply(complete). -- cgit v1.2.3 From 25238d8967ec52cfc8d132b1a2747ba587787e04 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 7 Jun 2010 09:32:02 -0400 Subject: cleanup after we're done --- src/fabric_util.erl | 5 ++++- src/fabric_view_all_docs.erl | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 760e511c..f0ab213d 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -1,6 +1,6 @@ -module(fabric_util). --export([submit_jobs/3, recv/4, receive_loop/4, receive_loop/6]). +-export([submit_jobs/3, cleanup/1, recv/4, receive_loop/4, receive_loop/6]). -include("fabric.hrl"). @@ -11,6 +11,9 @@ submit_jobs(Shards, EndPoint, ExtraArgs) -> Shard#shard{ref = Ref} end, Shards). +cleanup(Workers) -> + [rexi:kill(Node, Ref) || #shard{node=Node, ref=Ref} <- Workers]. + recv(Workers, Keypos, Fun, Acc0) -> receive_loop(Workers, Keypos, Fun, Acc0). diff --git a/src/fabric_view_all_docs.erl b/src/fabric_view_all_docs.erl index b91f0665..6cdc66c5 100644 --- a/src/fabric_view_all_docs.erl +++ b/src/fabric_view_all_docs.erl @@ -21,12 +21,14 @@ go(DbName, #view_query_args{keys=nil} = QueryArgs, Callback, Acc0) -> limit = Limit, user_acc = Acc0 }, - case fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, + try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, State, infinity, 5000) of {ok, NewState} -> {ok, NewState#collector.user_acc}; Error -> Error + after + fabric_util:cleanup(Workers) end; go(DbName, QueryArgs, Callback, Acc0) -> -- cgit v1.2.3 From 3d4fdbbfa1393ea7e3ed8898164852ffbe139576 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 7 Jun 2010 10:40:14 -0400 Subject: partial support for #user_ctx in fabric RPC calls, BugzID 10277 --- src/fabric.erl | 25 +++++++++++++++++++------ src/fabric_rpc.erl | 14 +++++++------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index b493d55c..29af3224 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -36,29 +36,29 @@ get_doc_count(DbName) -> fabric_db_doc_count:go(dbname(DbName)). create_db(DbName, Options) -> - fabric_db_create:create_db(dbname(DbName), Options). + fabric_db_create:create_db(dbname(DbName), opts(Options)). delete_db(DbName, Options) -> - fabric_db_delete:delete_db(dbname(DbName), Options). + fabric_db_delete:delete_db(dbname(DbName), opts(Options)). open_doc(DbName, Id, Options) -> - fabric_doc_open:go(dbname(DbName), docid(Id), Options). + fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options)). open_revs(DbName, Id, Revs, Options) -> - fabric_doc_open_revs:go(dbname(DbName), docid(Id), Revs, Options). + fabric_doc_open_revs:go(dbname(DbName), docid(Id), Revs, opts(Options)). get_missing_revs(DbName, IdsRevs) when is_list(IdsRevs) -> Sanitized = [idrevs(IdR) || IdR <- IdsRevs], fabric_doc_missing_revs:go(dbname(DbName), Sanitized). update_doc(DbName, Doc, Options) -> - {ok, [Result]} = update_docs(DbName, [Doc], Options), + {ok, [Result]} = update_docs(DbName, [Doc], opts(Options)), Result. update_docs(DbName, Docs, Options) -> - fabric_doc_update:go(dbname(DbName), docs(Docs), Options). + fabric_doc_update:go(dbname(DbName), docs(Docs), opts(Options)). all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when @@ -101,6 +101,19 @@ rev(Rev) when is_list(Rev); is_binary(Rev) -> rev({Seq, Hash} = Rev) when is_integer(Seq), is_binary(Hash) -> Rev. +opts(Options) -> + case couch_util:get_value(user_ctx, Options) of + undefined -> + case erlang:get(user_ctx) of + #user_ctx{} = Ctx -> + [{user_ctx, Ctx} | Options]; + _ -> + Options + end; + _ -> + Options + end. + generate_customer_path("/", _Customer) -> ""; generate_customer_path("/favicon.ico", _Customer) -> diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 81a8ba48..a2d42007 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -40,10 +40,10 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> final_all_docs_response(Db, Acc#view_acc.offset). get_db_info(DbName) -> - with_db(DbName, {couch_db, get_db_info, []}). + with_db(DbName, [], {couch_db, get_db_info, []}). get_doc_count(DbName) -> - with_db(DbName, {couch_db, get_doc_count, []}). + with_db(DbName, [], {couch_db, get_doc_count, []}). get_update_seq(DbName) -> rexi:reply(case couch_db:open(DbName, []) of @@ -54,10 +54,10 @@ get_update_seq(DbName) -> end). open_doc(DbName, DocId, Options) -> - with_db(DbName, {couch_db, open_doc_int, [DocId, Options]}). + with_db(DbName, Options, {couch_db, open_doc_int, [DocId, Options]}). open_revs(DbName, Id, Revs, Options) -> - with_db(DbName, {couch_db, open_doc_revs, [Id, Revs, Options]}). + with_db(DbName, Options, {couch_db, open_doc_revs, [Id, Revs, Options]}). get_missing_revs(DbName, IdRevsList) -> % reimplement here so we get [] for Ids with no missing revs in response @@ -77,14 +77,14 @@ get_missing_revs(DbName, IdRevsList) -> end). update_docs(DbName, Docs, Options) -> - with_db(DbName, {couch_db, update_docs, [Docs, Options]}). + with_db(DbName, Options, {couch_db, update_docs, [Docs, Options]}). %% %% internal %% -with_db(DbName, {M,F,A}) -> - case couch_db:open(DbName, []) of +with_db(DbName, Options, {M,F,A}) -> + case couch_db:open(DbName, Options) of {ok, Db} -> rexi:reply(apply(M, F, [Db | A])); Error -> -- cgit v1.2.3 From a9a36b4184fae5e4078982d48c0536b8ba948391 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 7 Jun 2010 10:44:26 -0400 Subject: primitive design_docs resource, BugzID 10243 --- src/fabric.erl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/fabric.erl b/src/fabric.erl index 29af3224..29396e9e 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -12,7 +12,7 @@ -export([all_docs/4]). % miscellany --export([db_path/2]). +-export([db_path/2, design_docs/1]). -include("fabric.hrl"). @@ -65,6 +65,22 @@ all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when is_function(Callback, 2) -> fabric_view_all_docs:go(dbname(DbName), QueryArgs, Callback, Acc0). +design_docs(DbName) -> + QueryArgs = #view_query_args{start_key = <<"_design/">>, include_docs=true}, + Callback = fun({total_and_offset, _, _}, []) -> + {ok, []}; + ({row, {Props}}, Acc) -> + case couch_util:get_value(id, Props) of + <<"_design/", _/binary>> -> + {ok, [couch_util:get_value(doc, Props) | Acc]}; + _ -> + {stop, Acc} + end; + (complete, Acc) -> + {ok, lists:reverse(Acc)} + end, + fabric:all_docs(dbname(DbName), QueryArgs, Callback, []). + %% some simple type validation and transcoding dbname(DbName) when is_list(DbName) -> -- cgit v1.2.3 From 38bde53e2429f60b3209d71c562218f3c7429945 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 7 Jun 2010 14:46:35 -0400 Subject: map views w/o keylist, BugzID 10220 --- ebin/fabric.app | 5 +- include/fabric.hrl | 1 + src/fabric.erl | 12 ++- src/fabric_rpc.erl | 105 +++++++++++++++++++++-- src/fabric_view_all_docs.erl | 199 +++++-------------------------------------- 5 files changed, 135 insertions(+), 187 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index 2e0f8df3..c61ba87a 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -10,13 +10,16 @@ fabric_db_delete, fabric_db_doc_count, fabric_db_info, + fabric_dict, fabric_doc_missing_revs, fabric_doc_open, fabric_doc_open_revs, fabric_doc_update, fabric_rpc, fabric_util, - fabric_view_all_docs + fabric_view, + fabric_view_all_docs, + fabric_view_map ]}, {registered, []}, {included_applications, []}, diff --git a/include/fabric.hrl b/include/fabric.hrl index 31e4336c..fa8319b4 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -21,6 +21,7 @@ rows = [], skip, limit, + stop_fun, user_acc }). diff --git a/src/fabric.erl b/src/fabric.erl index 29396e9e..983cd818 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -9,7 +9,7 @@ update_docs/3]). % Views --export([all_docs/4]). +-export([all_docs/4, query_view/5]). % miscellany -export([db_path/2, design_docs/1]). @@ -65,6 +65,12 @@ all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when is_function(Callback, 2) -> fabric_view_all_docs:go(dbname(DbName), QueryArgs, Callback, Acc0). +query_view(DbName, View, #view_query_args{view_type=reduce} = QueryArgs, + Callback, Acc0) -> + fabric_view_reduce:go(dbname(DbName), view(View), QueryArgs, Callback, Acc0); +query_view(DbName, View, #view_query_args{} = QueryArgs, Callback, Acc0) -> + fabric_view_map:go(dbname(DbName), view(View), QueryArgs, Callback, Acc0). + design_docs(DbName) -> QueryArgs = #view_query_args{start_key = <<"_design/">>, include_docs=true}, Callback = fun({total_and_offset, _, _}, []) -> @@ -117,6 +123,10 @@ rev(Rev) when is_list(Rev); is_binary(Rev) -> rev({Seq, Hash} = Rev) when is_integer(Seq), is_binary(Hash) -> Rev. +view(ViewName) -> + [Group, View] = re:split(ViewName, "/"), + {Group, View}. + opts(Options) -> case couch_util:get_value(user_ctx, Options) of undefined -> diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index a2d42007..0eab5e6d 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -2,7 +2,7 @@ -export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). --export([all_docs/2]). +-export([all_docs/2, map_view/4]). -include("fabric.hrl"). @@ -11,6 +11,7 @@ limit, include_docs, offset = nil, + total_rows, reduce_fun = fun couch_db:enum_docs_reduce_to_count/1, stop_fun, group_level = 0 @@ -30,14 +31,46 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> direction = Dir } = QueryArgs, StartId = if is_binary(StartKey) -> StartKey; true -> StartDocId end, + {ok, Total} = couch_db:get_doc_count(Db), Acc0 = #view_acc{ db = Db, include_docs = IncludeDocs, limit = Limit+Skip, + total_rows = Total, stop_fun = all_docs_stop_fun(QueryArgs) }, {ok, Acc} = couch_db:enum_docs(Db, StartId, Dir, fun view_fold/3, Acc0), - final_all_docs_response(Db, Acc#view_acc.offset). + final_response(Total, Acc#view_acc.offset). + +map_view(DbName, DDoc, ViewName, #view_query_args{keys=nil} = QueryArgs) -> + {ok, Db} = couch_db:open(DbName, []), + #view_query_args{ + start_key = StartKey, + start_docid = StartDocId, + limit = Limit, + skip = Skip, + include_docs = IncludeDocs, + direction = Dir, + stale = Stale, + view_type = ViewType + } = QueryArgs, + Start = {StartKey, StartDocId}, + MinSeq = if Stale == ok -> 0; true -> couch_db:get_update_seq(Db) end, + Group0 = couch_view_group:design_doc_to_view_group(Db, DDoc), + {ok, Pid} = gen_server:call(couch_view, {get_group_server, DbName, Group0}), + {ok, Group} = couch_view_group:request_group(Pid, MinSeq), + View = extract_view(Pid, ViewName, Group#group.views, ViewType), + {ok, Total} = couch_view:get_row_count(View), + Acc0 = #view_acc{ + db = Db, + include_docs = IncludeDocs, + limit = Limit+Skip, + total_rows = Total, + reduce_fun = fun couch_view:reduce_to_count/1, + stop_fun = default_stop_fun(QueryArgs) + }, + {ok, Acc} = couch_view:fold(View, Start, Dir, fun view_fold/3, Acc0), + final_response(Total, Acc#view_acc.offset). get_db_info(DbName) -> with_db(DbName, [], {couch_db, get_db_info, []}). @@ -101,10 +134,9 @@ view_fold(#full_doc_info{} = FullDocInfo, OffsetReds, Acc) -> #doc_info{revs=[#rev_info{deleted=true}|_]} -> {ok, Acc} end; -view_fold(KV, OffsetReds, #view_acc{offset=nil} = Acc) -> +view_fold(KV, OffsetReds, #view_acc{offset=nil, total_rows=Total} = Acc) -> % calculates the offset for this shard - #view_acc{db=Db, reduce_fun=Reduce} = Acc, - {ok, Total} = couch_db:get_doc_count(Db), + #view_acc{reduce_fun=Reduce} = Acc, Offset = Reduce(OffsetReds), case rexi:sync_reply({total_and_offset, Total, Offset}) of ok -> @@ -150,10 +182,67 @@ all_docs_stop_fun(#view_query_args{direction=rev, end_key=EndKey}) -> couch_db_updater:less_docid(ViewKey, EndKey) end. -final_all_docs_response(Db, nil) -> - {ok, Total} = couch_db:get_doc_count(Db), +final_response(Total, nil) -> case rexi:sync_reply({total_and_offset, Total, Total}) of ok -> rexi:reply(complete); stop -> ok end; -final_all_docs_response(_Db, _Offset) -> +final_response(_Total, _Offset) -> rexi:reply(complete). + +default_stop_fun(#view_query_args{direction=fwd, inclusive_end=true} = Args) -> + #view_query_args{end_key=EndKey, end_docid=EndDocId} = Args, + fun(ViewKey, ViewId) -> + couch_view:less_json([EndKey, EndDocId], [ViewKey, ViewId]) + end; +default_stop_fun(#view_query_args{direction=fwd} = Args) -> + #view_query_args{end_key=EndKey, end_docid=EndDocId} = Args, + fun + (ViewKey, _ViewId) when ViewKey == EndKey -> + true; + (ViewKey, ViewId) -> + couch_view:less_json([EndKey, EndDocId], [ViewKey, ViewId]) + end; +default_stop_fun(#view_query_args{direction=rev, inclusive_end=true} = Args) -> + #view_query_args{end_key=EndKey, end_docid=EndDocId} = Args, + fun(ViewKey, ViewId) -> + couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId]) + end; +default_stop_fun(#view_query_args{direction=rev} = Args) -> + #view_query_args{end_key=EndKey, end_docid=EndDocId} = Args, + fun + (ViewKey, _ViewId) when ViewKey == EndKey -> + true; + (ViewKey, ViewId) -> + couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId]) + end. + +extract_view(Pid, ViewName, [], _ViewType) -> + ?LOG_ERROR("missing_named_view ~p", [ViewName]), + exit(Pid, kill), + exit(missing_named_view); +extract_view(Pid, ViewName, [View|Rest], ViewType) -> + case lists:member(ViewName, view_names(View, ViewType)) of + true -> + if ViewType == reduce -> + {index_of(ViewName, view_names(View, reduce)), View}; + true -> + View + end; + false -> + extract_view(Pid, ViewName, Rest, ViewType) + end. + +view_names(View, Type) when Type == red_map; Type == reduce -> + [Name || {Name, _} <- View#view.reduce_funs]; +view_names(View, map) -> + View#view.map_names. + +index_of(X, List) -> + index_of(X, List, 1). + +index_of(_X, [], _I) -> + not_found; +index_of(X, [X|_Rest], I) -> + I; +index_of(X, [_|Rest], I) -> + index_of(X, Rest, I+1). diff --git a/src/fabric_view_all_docs.erl b/src/fabric_view_all_docs.erl index 6cdc66c5..99834286 100644 --- a/src/fabric_view_all_docs.erl +++ b/src/fabric_view_all_docs.erl @@ -16,9 +16,10 @@ go(DbName, #view_query_args{keys=nil} = QueryArgs, Callback, Acc0) -> query_args = QueryArgs, callback = Callback, buffer_size = list_to_integer(BufferSize), - counters = init_counters(Workers), + counters = fabric_dict:init(Workers, 0), skip = Skip, limit = Limit, + stop_fun = stop_fun(QueryArgs), user_acc = Acc0 }, try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, @@ -58,8 +59,8 @@ handle_message({rexi_DOWN, _, _, _}, nil, State) -> handle_message({rexi_EXIT, _}, Worker, State) -> #collector{callback=Callback, counters=Counters0, user_acc=Acc} = State, - Counters = remove(Worker, Counters0), - case is_progress_possible(Counters) of + Counters = fabric_dict:erase(Worker, Counters0), + case fabric_view:is_progress_possible(Counters) of true -> {ok, State#collector{counters = Counters}}; false -> @@ -75,18 +76,18 @@ handle_message({total_and_offset, Tot, Off}, {Worker, From}, State) -> offset = Offset0, user_acc = AccIn } = State, - case lookup_element(Worker, Counters0) of + case fabric_dict:lookup_element(Worker, Counters0) of undefined -> % this worker lost the race with other partition copies, terminate gen_server:reply(From, stop), {ok, State}; 0 -> gen_server:reply(From, ok), - Counters1 = update_counter(Worker, 1, Counters0), - Counters2 = remove_overlapping_shards(Worker, Counters1), + Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), + Counters2 = fabric_view:remove_overlapping_shards(Worker, Counters1), Total = Total0 + Tot, Offset = Offset0 + Off, - case waiting_on_shards(Counters2) of + case fabric_dict:any(0, Counters2) of true -> {ok, State#collector{ counters = Counters2, @@ -97,7 +98,7 @@ handle_message({total_and_offset, Tot, Off}, {Worker, From}, State) -> FinalOffset = erlang:min(Total, Offset+State#collector.skip), {Go, Acc} = Callback({total_and_offset, Total, FinalOffset}, AccIn), {Go, State#collector{ - counters = decrement_all_counters(Counters2), + counters = fabric_dict:decrement_all(Counters2), total_rows = Total, offset = FinalOffset, user_acc = Acc @@ -109,158 +110,30 @@ handle_message(#view_row{} = Row, {Worker, From}, State) -> #collector{query_args = Args, counters = Counters0, rows = Rows0} = State, Dir = Args#view_query_args.direction, Rows = merge_row(Dir, Row#view_row{worker=Worker}, Rows0), - Counters1 = update_counter(Worker, 1, Counters0), + Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), State1 = State#collector{rows=Rows, counters=Counters1}, - State2 = maybe_pause_worker(Worker, From, State1), - maybe_send_row(State2); + State2 = fabric_view:maybe_pause_worker(Worker, From, State1), + fabric_view:maybe_send_row(State2); handle_message(complete, Worker, State) -> - Counters = update_counter(Worker, 1, State#collector.counters), - maybe_send_row(State#collector{counters = Counters}). + Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters), + fabric_view:maybe_send_row(State#collector{counters = Counters}). -maybe_pause_worker(Worker, From, State) -> - #collector{buffer_size = BufferSize, counters = Counters} = State, - case lookup_element(Worker, Counters) of - BufferSize -> - State#collector{blocked = [{Worker,From} | State#collector.blocked]}; - _Count -> - gen_server:reply(From, ok), - State - end. - -maybe_resume_worker(Worker, State) -> - #collector{buffer_size = Buffer, counters = C, blocked = B} = State, - case lookup_element(Worker, C) of - Count when Count < Buffer/2 -> - case couch_util:get_value(Worker, B) of - undefined -> - State; - From -> - gen_server:reply(From, ok), - State#collector{blocked = lists:keydelete(Worker, 1, B)} - end; - _Other -> - State - end. - -maybe_send_row(#collector{limit=0} = State) -> - #collector{user_acc=AccIn, callback=Callback} = State, - {_, Acc} = Callback(complete, AccIn), - {stop, State#collector{user_acc=Acc}}; -maybe_send_row(State) -> - #collector{ - callback = Callback, - counters = Counters, - skip = Skip, - limit = Limit, - user_acc = AccIn - } = State, - case waiting_on_shards(Counters) of - true -> - {ok, State}; - false -> - case get_next_row(State) of - complete -> - {_, Acc} = Callback(complete, AccIn), - {stop, State#collector{user_acc=Acc}}; - {_, NewState} when Skip > 0 -> - maybe_send_row(NewState#collector{skip=Skip-1, limit=Limit-1}); - {Row, NewState} -> - case Callback(transform_row(Row), AccIn) of - {stop, Acc} -> - {stop, NewState#collector{user_acc=Acc, limit=Limit-1}}; - {ok, Acc} -> - maybe_send_row(NewState#collector{user_acc=Acc, limit=Limit-1}) - end - end - end. - -get_next_row(#collector{rows = []}) -> - complete; -get_next_row(State) -> - #collector{query_args=Args, rows=[Row|Rest], counters=Counters0} = State, - Worker = Row#view_row.worker, - Counters1 = update_counter(Worker, -1, Counters0), - NewState = maybe_resume_worker(Worker, State#collector{counters=Counters1}), - case stop(Args, Row) of - true -> - complete; - false -> - {Row, NewState#collector{rows = Rest}} +stop_fun(#view_query_args{direction=fwd, end_key=EndKey}) -> + fun(#view_row{id=Id}) -> + couch_db_updater:less_docid(EndKey, Id) + end; +stop_fun(#view_query_args{direction=rev, end_key=EndKey}) -> + fun(#view_row{id=Id}) -> + couch_db_updater:less_docid(Id, EndKey) end. -stop(#view_query_args{direction=fwd, end_key=EndKey}, #view_row{id=Id}) -> - couch_db_updater:less_docid(EndKey, Id); -stop(#view_query_args{direction=rev, end_key=EndKey}, #view_row{id=Id}) -> - couch_db_updater:less_docid(Id, EndKey). - -transform_row(#view_row{key=Key, id=undefined}) -> - {row, {[{key,Key}, {error,not_found}]}}; -transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}]}}; -transform_row(#view_row{key=Key, id=Id, value=Value, doc={error,Reason}}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}, {error,Reason}]}}; -transform_row(#view_row{key=Key, id=Id, value=Value, doc=Doc}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}, {doc,Doc}]}}. - merge_row(fwd, Row, Rows) -> lists:keymerge(#view_row.id, [Row], Rows); merge_row(rev, Row, Rows) -> lists:rkeymerge(#view_row.id, [Row], Rows). -remove_overlapping_shards(#shard{range=[A,B]} = Shard0, Shards) -> - filter(fun(#shard{range=[X,Y]} = Shard, _Value) -> - if Shard =:= Shard0 -> - % we can't remove ourselves - true; - A < B, X >= A, X < B -> - % lower bound is inside our range - false; - A < B, Y > A, Y =< B -> - % upper bound is inside our range - false; - B < A, X >= A orelse B < A, X < B -> - % target shard wraps the key range, lower bound is inside - false; - B < A, Y > A orelse B < A, Y =< B -> - % target shard wraps the key range, upper bound is inside - false; - true -> - true - end - end, Shards). - -%% @doc looks for a fully covered keyrange in the list of counters --spec is_progress_possible([{#shard{}, non_neg_integer()}]) -> boolean(). -is_progress_possible(Counters) -> - Ranges = fold(fun(#shard{range=[X,Y]}, _, A) -> [{X,Y}|A] end, [], Counters), - [First | Rest] = lists:ukeysort(1, Ranges), - {Head, Tail} = lists:foldl(fun - (_, {Head, Tail}) when Head =:= Tail -> - % this is the success condition, we can fast-forward - {Head, Tail}; - (_, {foo, bar}) -> - % we've already declared failure - {foo, bar}; - ({X,_}, {Head, Tail}) when Head < Tail, X > Tail -> - % gap in the keyrange, we're dead - {foo, bar}; - ({X,Y}, {Head, Tail}) when Head < Tail, X < Y -> - % the normal condition, adding to the tail - {Head, erlang:max(Tail, Y)}; - ({X,Y}, {Head, Tail}) when Head < Tail, X > Y, Y >= Head -> - % we've wrapped all the way around, trigger success condition - {Head, Head}; - ({X,Y}, {Head, Tail}) when Head < Tail, X > Y -> - % this wraps the keyspace, but there's still a gap. We're dead - % TODO technically, another shard could be a superset of this one, and - % we could still be alive. Pretty unlikely though, and impossible if - % we don't allow shards to wrap around the boundary - {foo, bar} - end, First, Rest), - Head =:= Tail. - doc_receive_loop([], _, _, _, Acc) -> {ok, Acc}; doc_receive_loop(_, _, 0, _, Acc) -> @@ -273,7 +146,7 @@ doc_receive_loop([{Pid,Ref}|Rest], Skip, Limit, Callback, Acc) when Skip > 0 -> end; doc_receive_loop([{Pid,Ref}|Rest], 0, Limit, Callback, AccIn) -> receive {'DOWN', Ref, process, Pid, #view_row{} = Row} -> - case Callback(transform_row(Row), AccIn) of + case Callback(fabric_view:transform_row(Row), AccIn) of {ok, Acc} -> doc_receive_loop(Rest, 0, Limit-1, Callback, Acc); {stop, Acc} -> @@ -300,31 +173,3 @@ open_doc(DbName, Id, IncludeDocs) -> #view_row{key=Id, id=Id, value=Value} end, exit(if IncludeDocs -> Row#view_row{doc=Doc}; true -> Row end). - - -% Instead of ets, let's use an ordered keylist. We'll need to revisit if we -% have >> 100 shards, so a private interface is a good idea. - APK June 2010 - -init_counters(Keys) -> - orddict:from_list([{Key,0} || Key <- Keys]). - -decrement_all_counters(Dict) -> - [{K,V-1} || {K,V} <- Dict]. - -update_counter(Key, Incr, Dict0) -> - orddict:update_counter(Key, Incr, Dict0). - -lookup_element(Key, Dict) -> - couch_util:get_value(Key, Dict). - -waiting_on_shards(Dict) -> - lists:keymember(0, 2, Dict). - -remove(Shard, Dict) -> - orddict:erase(Shard, Dict). - -filter(Fun, Dict) -> - orddict:filter(Fun, Dict). - -fold(Fun, Acc0, Dict) -> - orddict:fold(Fun, Acc0, Dict). -- cgit v1.2.3 From ab14b5bfdb88d9f07b6885f4cc1208f80c8c72f4 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 7 Jun 2010 16:03:51 -0400 Subject: map views w/ keys, also forgot to git add stuff, BugzID 10220 --- include/fabric.hrl | 1 + src/fabric_dict.erl | 32 ++++++++++ src/fabric_rpc.erl | 17 +++++- src/fabric_view.erl | 153 ++++++++++++++++++++++++++++++++++++++++++++++++ src/fabric_view_map.erl | 148 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 src/fabric_dict.erl create mode 100644 src/fabric_view.erl create mode 100644 src/fabric_view_map.erl diff --git a/include/fabric.hrl b/include/fabric.hrl index fa8319b4..460cf578 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -22,6 +22,7 @@ skip, limit, stop_fun, + keydict, user_acc }). diff --git a/src/fabric_dict.erl b/src/fabric_dict.erl new file mode 100644 index 00000000..a4191682 --- /dev/null +++ b/src/fabric_dict.erl @@ -0,0 +1,32 @@ +-module(fabric_dict). +-compile(export_all). + +% Instead of ets, let's use an ordered keylist. We'll need to revisit if we +% have >> 100 shards, so a private interface is a good idea. - APK June 2010 + +init(Keys, InitialValue) -> + orddict:from_list([{Key, InitialValue} || Key <- Keys]). + + +decrement_all(Dict) -> + [{K,V-1} || {K,V} <- Dict]. + +erase(Key, Dict) -> + orddict:erase(Key, Dict). + +update_counter(Key, Incr, Dict0) -> + orddict:update_counter(Key, Incr, Dict0). + + +lookup_element(Key, Dict) -> + couch_util:get_value(Key, Dict). + + +any(Value, Dict) -> + lists:keymember(Value, 2, Dict). + +filter(Fun, Dict) -> + orddict:filter(Fun, Dict). + +fold(Fun, Acc0, Dict) -> + orddict:fold(Fun, Acc0, Dict). diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 0eab5e6d..85c01906 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -42,13 +42,14 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> {ok, Acc} = couch_db:enum_docs(Db, StartId, Dir, fun view_fold/3, Acc0), final_response(Total, Acc#view_acc.offset). -map_view(DbName, DDoc, ViewName, #view_query_args{keys=nil} = QueryArgs) -> +map_view(DbName, DDoc, ViewName, QueryArgs) -> {ok, Db} = couch_db:open(DbName, []), #view_query_args{ start_key = StartKey, start_docid = StartDocId, limit = Limit, skip = Skip, + keys = Keys, include_docs = IncludeDocs, direction = Dir, stale = Stale, @@ -69,7 +70,19 @@ map_view(DbName, DDoc, ViewName, #view_query_args{keys=nil} = QueryArgs) -> reduce_fun = fun couch_view:reduce_to_count/1, stop_fun = default_stop_fun(QueryArgs) }, - {ok, Acc} = couch_view:fold(View, Start, Dir, fun view_fold/3, Acc0), + case Keys of + nil -> + {ok, Acc} = couch_view:fold(View, Start, Dir, fun view_fold/3, Acc0); + _ -> + Acc = lists:foldl(fun(Key, AccIn) -> + KeyStart = {Key, StartDocId}, + KeyStop = default_stop_fun(QueryArgs#view_query_args{start_key=Key, + end_key=Key}), + {_Go, Out} = couch_view:fold(View, KeyStart, Dir, fun view_fold/3, + AccIn#view_acc{stop_fun = KeyStop}), + Out + end, Acc0, Keys) + end, final_response(Total, Acc#view_acc.offset). get_db_info(DbName) -> diff --git a/src/fabric_view.erl b/src/fabric_view.erl new file mode 100644 index 00000000..ae5ce361 --- /dev/null +++ b/src/fabric_view.erl @@ -0,0 +1,153 @@ +-module(fabric_view). + +-export([is_progress_possible/1, remove_overlapping_shards/2, maybe_send_row/1, + maybe_pause_worker/3, maybe_resume_worker/2, transform_row/1, keydict/1]). + +-include("fabric.hrl"). + +%% @doc looks for a fully covered keyrange in the list of counters +-spec is_progress_possible([{#shard{}, non_neg_integer()}]) -> boolean(). +is_progress_possible(Counters) -> + Ranges = fabric_dict:fold(fun(#shard{range=[X,Y]}, _, A) -> [{X,Y}|A] end, + [], Counters), + [First | Rest] = lists:ukeysort(1, Ranges), + {Head, Tail} = lists:foldl(fun + (_, {Head, Tail}) when Head =:= Tail -> + % this is the success condition, we can fast-forward + {Head, Tail}; + (_, {foo, bar}) -> + % we've already declared failure + {foo, bar}; + ({X,_}, {Head, Tail}) when Head < Tail, X > Tail -> + % gap in the keyrange, we're dead + {foo, bar}; + ({X,Y}, {Head, Tail}) when Head < Tail, X < Y -> + % the normal condition, adding to the tail + {Head, erlang:max(Tail, Y)}; + ({X,Y}, {Head, Tail}) when Head < Tail, X > Y, Y >= Head -> + % we've wrapped all the way around, trigger success condition + {Head, Head}; + ({X,Y}, {Head, Tail}) when Head < Tail, X > Y -> + % this wraps the keyspace, but there's still a gap. We're dead + % TODO technically, another shard could be a superset of this one, and + % we could still be alive. Pretty unlikely though, and impossible if + % we don't allow shards to wrap around the boundary + {foo, bar} + end, First, Rest), + Head =:= Tail. + +-spec remove_overlapping_shards(#shard{}, [#shard{}]) -> [#shard{}]. +remove_overlapping_shards(#shard{range=[A,B]} = Shard0, Shards) -> + fabric_dict:filter(fun(#shard{range=[X,Y]} = Shard, _Value) -> + if Shard =:= Shard0 -> + % we can't remove ourselves + true; + A < B, X >= A, X < B -> + % lower bound is inside our range + false; + A < B, Y > A, Y =< B -> + % upper bound is inside our range + false; + B < A, X >= A orelse B < A, X < B -> + % target shard wraps the key range, lower bound is inside + false; + B < A, Y > A orelse B < A, Y =< B -> + % target shard wraps the key range, upper bound is inside + false; + true -> + true + end + end, Shards). + +maybe_pause_worker(Worker, From, State) -> + #collector{buffer_size = BufferSize, counters = Counters} = State, + case fabric_dict:lookup_element(Worker, Counters) of + BufferSize -> + State#collector{blocked = [{Worker,From} | State#collector.blocked]}; + _Count -> + gen_server:reply(From, ok), + State + end. + +maybe_resume_worker(Worker, State) -> + #collector{buffer_size = Buffer, counters = C, blocked = B} = State, + case fabric_dict:lookup_element(Worker, C) of + Count when Count < Buffer/2 -> + case couch_util:get_value(Worker, B) of + undefined -> + State; + From -> + gen_server:reply(From, ok), + State#collector{blocked = lists:keydelete(Worker, 1, B)} + end; + _Other -> + State + end. + +maybe_send_row(#collector{limit=0} = State) -> + #collector{user_acc=AccIn, callback=Callback} = State, + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc=Acc}}; +maybe_send_row(State) -> + #collector{ + callback = Callback, + counters = Counters, + skip = Skip, + limit = Limit, + user_acc = AccIn + } = State, + case fabric_dict:any(0, Counters) of + true -> + {ok, State}; + false -> + case get_next_row(State) of + complete -> + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc=Acc}}; + {_, NewState} when Skip > 0 -> + maybe_send_row(NewState#collector{skip=Skip-1, limit=Limit-1}); + {Row, NewState} -> + case Callback(transform_row(Row), AccIn) of + {stop, Acc} -> + {stop, NewState#collector{user_acc=Acc, limit=Limit-1}}; + {ok, Acc} -> + maybe_send_row(NewState#collector{user_acc=Acc, limit=Limit-1}) + end + end + end. + +keydict(nil) -> + undefined; +keydict(Keys) -> + {Dict,_} = lists:foldl(fun(K, {D,I}) -> {dict:store(K,I,D), I+1} end, + {dict:new(),0}, Keys), + Dict. + +%% internal %% + +get_next_row(#collector{rows = []}) -> + complete; +get_next_row(State) -> + #collector{ + rows = [Row|Rest], + counters = Counters0, + stop_fun = Stop + } = State, + Worker = Row#view_row.worker, + Counters1 = fabric_dict:update_counter(Worker, -1, Counters0), + NewState = maybe_resume_worker(Worker, State#collector{counters=Counters1}), + case Stop(Row) of + true -> + complete; + false -> + {Row, NewState#collector{rows = Rest}} + end. + +transform_row(#view_row{key=Key, id=undefined}) -> + {row, {[{key,Key}, {error,not_found}]}}; +transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}]}}; +transform_row(#view_row{key=Key, id=Id, value=Value, doc={error,Reason}}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}, {error,Reason}]}}; +transform_row(#view_row{key=Key, id=Id, value=Value, doc=Doc}) -> + {row, {[{key,Key}, {id,Id}, {value,Value}, {doc,Doc}]}}. diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl new file mode 100644 index 00000000..6ec7dfde --- /dev/null +++ b/src/fabric_view_map.erl @@ -0,0 +1,148 @@ +-module(fabric_view_map). + +-export([go/5]). + +-include("fabric.hrl"). + +go(DbName, {GroupId, View}, Args, Callback, Acc0) -> + {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), + Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> + Ref = rexi:cast(Node, {fabric_rpc, map_view, [Name, DDoc, View, Args]}), + Shard#shard{ref = Ref} + end, partitions:all_parts(DbName)), + BufferSize = couch_config:get("fabric", "map_buffer_size", "2"), + #view_query_args{limit = Limit, skip = Skip, keys = Keys} = Args, + State = #collector{ + query_args = Args, + callback = Callback, + buffer_size = list_to_integer(BufferSize), + counters = fabric_dict:init(Workers, 0), + skip = Skip, + limit = Limit, + stop_fun = stop_fun(Args), + keydict = fabric_view:keydict(Keys), + user_acc = Acc0 + }, + try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, + State, infinity, 5000) of + {ok, NewState} -> + {ok, NewState#collector.user_acc}; + Error -> + Error + after + fabric_util:cleanup(Workers) + end. + +handle_message({rexi_DOWN, _, _, _}, nil, State) -> + % TODO see if progress can be made here, possibly by removing all shards + % from that node and checking is_progress_possible + {ok, State}; + +handle_message({rexi_EXIT, Reason}, Worker, State) -> + ?LOG_ERROR("~p rexi_EXIT ~p", [?MODULE, Reason]), + #collector{callback=Callback, counters=Counters0, user_acc=Acc} = State, + Counters = fabric_dict:erase(Worker, Counters0), + case fabric_view:is_progress_possible(Counters) of + true -> + {ok, State#collector{counters = Counters}}; + false -> + Callback({error, dead_shards}, Acc), + {error, dead_shards} + end; + +handle_message({total_and_offset, Tot, Off}, {Worker, From}, State) -> + #collector{ + callback = Callback, + counters = Counters0, + total_rows = Total0, + offset = Offset0, + user_acc = AccIn + } = State, + case fabric_dict:lookup_element(Worker, Counters0) of + undefined -> + % this worker lost the race with other partition copies, terminate + gen_server:reply(From, stop), + {ok, State}; + 0 -> + gen_server:reply(From, ok), + Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), + Counters2 = fabric_view:remove_overlapping_shards(Worker, Counters1), + Total = Total0 + Tot, + Offset = Offset0 + Off, + case fabric_dict:any(0, Counters2) of + true -> + {ok, State#collector{ + counters = Counters2, + total_rows = Total, + offset = Offset + }}; + false -> + FinalOffset = erlang:min(Total, Offset+State#collector.skip), + {Go, Acc} = Callback({total_and_offset, Total, FinalOffset}, AccIn), + {Go, State#collector{ + counters = fabric_dict:decrement_all(Counters2), + total_rows = Total, + offset = FinalOffset, + user_acc = Acc + }} + end + end; + +handle_message(#view_row{} = Row, {Worker, From}, State) -> + #collector{ + query_args = #view_query_args{direction=Dir}, + counters = Counters0, + rows = Rows0, + keydict = KeyDict + } = State, + Rows = merge_row(Dir, KeyDict, Row#view_row{worker=Worker}, Rows0), + Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), + State1 = State#collector{rows=Rows, counters=Counters1}, + State2 = fabric_view:maybe_pause_worker(Worker, From, State1), + fabric_view:maybe_send_row(State2); + +handle_message(complete, Worker, State) -> + Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters), + fabric_view:maybe_send_row(State#collector{counters = Counters}). + +stop_fun(#view_query_args{} = QueryArgs) -> + #view_query_args{ + direction = Dir, + inclusive_end = Inclusive, + end_key = EndKey, + end_docid = EndDocId + } = QueryArgs, + stop_fun(Dir, Inclusive, EndKey, EndDocId). + +stop_fun(fwd, true, EndKey, EndDocId) -> + fun(#view_row{key=Key, id=Id}) -> + couch_view:less_json([EndKey, EndDocId], [Key, Id]) + end; +stop_fun(fwd, false, EndKey, EndDocId) -> + fun(#view_row{key=K}) when K==EndKey -> true; (#view_row{key=Key, id=Id}) -> + couch_view:less_json([EndKey, EndDocId], [Key, Id]) + end; +stop_fun(rev, true, EndKey, EndDocId) -> + fun(#view_row{key=Key, id=Id}) -> + couch_view:less_json([Key, Id], [EndKey, EndDocId]) + end; +stop_fun(rev, false, EndKey, EndDocId) -> + fun(#view_row{key=K}) when K==EndKey -> true; (#view_row{key=Key, id=Id}) -> + couch_view:less_json([Key, Id], [EndKey, EndDocId]) + end. + +merge_row(fwd, undefined, Row, Rows) -> + lists:merge(fun(#view_row{key=KeyA, id=IdA}, #view_row{key=KeyB, id=IdB}) -> + couch_view:less_json([KeyA, IdA], [KeyB, IdB]) + end, [Row], Rows); +merge_row(rev, undefined, Row, Rows) -> + lists:merge(fun(#view_row{key=KeyA, id=IdA}, #view_row{key=KeyB, id=IdB}) -> + couch_view:less_json([KeyB, IdB], [KeyA, IdA]) + end, [Row], Rows); +merge_row(_, KeyDict, Row, Rows) -> + lists:merge(fun(#view_row{key=A, id=IdA}, #view_row{key=B, id=IdB}) -> + if A =:= B -> IdA < IdB; true -> + dict:fetch(A, KeyDict) < dict:fetch(B, KeyDict) + end + end, [Row], Rows). + -- cgit v1.2.3 From 2569631e249cc8209858f590a349f314b7253f3e Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 7 Jun 2010 23:44:33 -0400 Subject: reduce views, BugzID 10220 --- ebin/fabric.app | 3 +- include/fabric.hrl | 5 ++- src/fabric_rpc.erl | 87 +++++++++++++++++++++++++++++--------------- src/fabric_view.erl | 88 +++++++++++++++++++++++++++++++++++++++++---- src/fabric_view_map.erl | 4 +-- src/fabric_view_reduce.erl | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 237 insertions(+), 40 deletions(-) create mode 100644 src/fabric_view_reduce.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index c61ba87a..ef05bb5d 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -19,7 +19,8 @@ fabric_util, fabric_view, fabric_view_all_docs, - fabric_view_map + fabric_view_map, + fabric_view_reduce ]}, {registered, []}, {included_applications, []}, diff --git a/include/fabric.hrl b/include/fabric.hrl index 460cf578..f4665ca8 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -22,7 +22,10 @@ skip, limit, stop_fun, - keydict, + keys, + os_proc, + reducer, + lang, user_acc }). diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 85c01906..aa922585 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -2,7 +2,7 @@ -export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). --export([all_docs/2, map_view/4]). +-export([all_docs/2, map_view/4, reduce_view/4]). -include("fabric.hrl"). @@ -60,7 +60,7 @@ map_view(DbName, DDoc, ViewName, QueryArgs) -> Group0 = couch_view_group:design_doc_to_view_group(Db, DDoc), {ok, Pid} = gen_server:call(couch_view, {get_group_server, DbName, Group0}), {ok, Group} = couch_view_group:request_group(Pid, MinSeq), - View = extract_view(Pid, ViewName, Group#group.views, ViewType), + View = fabric_view:extract_view(Pid, ViewName, Group#group.views, ViewType), {ok, Total} = couch_view:get_row_count(View), Acc0 = #view_acc{ db = Db, @@ -85,6 +85,38 @@ map_view(DbName, DDoc, ViewName, QueryArgs) -> end, final_response(Total, Acc#view_acc.offset). +reduce_view(DbName, Group0, ViewName, QueryArgs) -> + {ok, Db} = couch_db:open(DbName, []), + #view_query_args{ + start_key = StartKey, + start_docid = StartDocId, + end_key = EndKey, + end_docid = EndDocId, + group_level = GroupLevel, + limit = Limit, + skip = Skip, + keys = Keys, + direction = Dir, + stale = Stale + } = QueryArgs, + GroupFun = group_rows_fun(GroupLevel), + MinSeq = if Stale == ok -> 0; true -> couch_db:get_update_seq(Db) end, + {ok, Pid} = gen_server:call(couch_view, {get_group_server, DbName, Group0}), + {ok, #group{views=Views, def_lang=Lang}} = couch_view_group:request_group( + Pid, MinSeq), + {NthRed, View} = fabric_view:extract_view(Pid, ViewName, Views, reduce), + ReduceView = {reduce, NthRed, Lang, View}, + Acc0 = #view_acc{group_level = GroupLevel, limit = Limit+Skip}, + case Keys of + nil -> + couch_view:fold_reduce(ReduceView, Dir, {StartKey,StartDocId}, + {EndKey,EndDocId}, GroupFun, fun reduce_fold/3, Acc0); + _ -> + [couch_view:fold_reduce(ReduceView, Dir, {K,StartDocId}, {K,EndDocId}, + GroupFun, fun reduce_fold/3, Acc0) || K <- Keys] + end, + rexi:reply(complete). + get_db_info(DbName) -> with_db(DbName, [], {couch_db, get_db_info, []}). @@ -229,33 +261,30 @@ default_stop_fun(#view_query_args{direction=rev} = Args) -> couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId]) end. -extract_view(Pid, ViewName, [], _ViewType) -> - ?LOG_ERROR("missing_named_view ~p", [ViewName]), - exit(Pid, kill), - exit(missing_named_view); -extract_view(Pid, ViewName, [View|Rest], ViewType) -> - case lists:member(ViewName, view_names(View, ViewType)) of - true -> - if ViewType == reduce -> - {index_of(ViewName, view_names(View, reduce)), View}; - true -> - View - end; - false -> - extract_view(Pid, ViewName, Rest, ViewType) +group_rows_fun(exact) -> + fun({Key1,_}, {Key2,_}) -> Key1 == Key2 end; +group_rows_fun(0) -> + fun(_A, _B) -> true end; +group_rows_fun(GroupLevel) when is_integer(GroupLevel) -> + fun({[_|_] = Key1,_}, {[_|_] = Key2,_}) -> + lists:sublist(Key1, GroupLevel) == lists:sublist(Key2, GroupLevel); + ({Key1,_}, {Key2,_}) -> + Key1 == Key2 end. -view_names(View, Type) when Type == red_map; Type == reduce -> - [Name || {Name, _} <- View#view.reduce_funs]; -view_names(View, map) -> - View#view.map_names. - -index_of(X, List) -> - index_of(X, List, 1). +reduce_fold(_Key, _Red, #view_acc{limit=0} = Acc) -> + {stop, Acc}; +reduce_fold(_Key, Red, #view_acc{group_level=0} = Acc) -> + send(null, Red, Acc); +reduce_fold(Key, Red, #view_acc{group_level=exact} = Acc) -> + send(Key, Red, Acc); +reduce_fold(K, Red, #view_acc{group_level=I} = Acc) when I > 0, is_list(K) -> + send(lists:sublist(K, I), Red, Acc). -index_of(_X, [], _I) -> - not_found; -index_of(X, [X|_Rest], I) -> - I; -index_of(X, [_|Rest], I) -> - index_of(X, Rest, I+1). +send(Key, Value, #view_acc{limit=Limit} = Acc) -> + case rexi:sync_reply(#view_row{key=Key, value=Value}) of + ok -> + {ok, Acc#view_acc{limit=Limit-1}}; + stop -> + exit(normal) + end. diff --git a/src/fabric_view.erl b/src/fabric_view.erl index ae5ce361..70dedf27 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -1,7 +1,8 @@ -module(fabric_view). -export([is_progress_possible/1, remove_overlapping_shards/2, maybe_send_row/1, - maybe_pause_worker/3, maybe_resume_worker/2, transform_row/1, keydict/1]). + maybe_pause_worker/3, maybe_resume_worker/2, transform_row/1, keydict/1, + extract_view/4]). -include("fabric.hrl"). @@ -100,10 +101,7 @@ maybe_send_row(State) -> true -> {ok, State}; false -> - case get_next_row(State) of - complete -> - {_, Acc} = Callback(complete, AccIn), - {stop, State#collector{user_acc=Acc}}; + try get_next_row(State) of {_, NewState} when Skip > 0 -> maybe_send_row(NewState#collector{skip=Skip-1, limit=Limit-1}); {Row, NewState} -> @@ -113,6 +111,9 @@ maybe_send_row(State) -> {ok, Acc} -> maybe_send_row(NewState#collector{user_acc=Acc, limit=Limit-1}) end + catch complete -> + {_, Acc} = Callback(complete, AccIn), + {stop, State#collector{user_acc=Acc}} end end. @@ -125,8 +126,31 @@ keydict(Keys) -> %% internal %% +get_next_row(#collector{reducer = RedSrc} = St) when RedSrc =/= undefined -> + #collector{ + query_args = #view_query_args{direction=Dir}, + keys = Keys, + rows = RowDict, + os_proc = Proc, + counters = Counters0 + } = St, + {Key, RestKeys} = find_next_key(Keys, Dir, RowDict), + case dict:find(Key, RowDict) of + {ok, Records} -> + NewRowDict = dict:erase(Key, RowDict), + Counters = lists:foldl(fun(#view_row{worker=Worker}, CountersAcc) -> + fabric_dict:update_counter(Worker, -1, CountersAcc) + end, Counters0, Records), + Wrapped = [[V] || #view_row{value=V} <- Records], + {ok, [Reduced]} = couch_query_servers:rereduce(Proc, [RedSrc], Wrapped), + NewSt = St#collector{keys=RestKeys, rows=NewRowDict, counters=Counters}, + {#view_row{key=Key, id=reduced, value=Reduced}, NewSt}; + error -> + NewSt = St#collector{keys=RestKeys}, + {#view_row{key=Key, id=reduced, value={error, missing}}, NewSt} + end; get_next_row(#collector{rows = []}) -> - complete; + throw(complete); get_next_row(State) -> #collector{ rows = [Row|Rest], @@ -138,11 +162,25 @@ get_next_row(State) -> NewState = maybe_resume_worker(Worker, State#collector{counters=Counters1}), case Stop(Row) of true -> - complete; + throw(complete); false -> {Row, NewState#collector{rows = Rest}} end. +find_next_key(undefined, Dir, RowDict) -> + case lists:sort(sort_fun(Dir), dict:fetch_keys(RowDict)) of + [] -> + throw(complete); + [Key|_] -> + {Key, undefined} + end; +find_next_key([], _, _) -> + throw(complete); +find_next_key([Key|Rest], _, _) -> + {Key, Rest}. + +transform_row(#view_row{key=Key, id=reduced, value=Value}) -> + {row, {[{key,Key}, {value,Value}]}}; transform_row(#view_row{key=Key, id=undefined}) -> {row, {[{key,Key}, {error,not_found}]}}; transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> @@ -151,3 +189,39 @@ transform_row(#view_row{key=Key, id=Id, value=Value, doc={error,Reason}}) -> {row, {[{key,Key}, {id,Id}, {value,Value}, {error,Reason}]}}; transform_row(#view_row{key=Key, id=Id, value=Value, doc=Doc}) -> {row, {[{key,Key}, {id,Id}, {value,Value}, {doc,Doc}]}}. + +sort_fun(fwd) -> + fun(A,A) -> true; (A,B) -> couch_view:less_json(A,B) end; +sort_fun(rev) -> + fun(A,A) -> true; (A,B) -> couch_view:less_json(B,A) end. + +extract_view(Pid, ViewName, [], _ViewType) -> + ?LOG_ERROR("missing_named_view ~p", [ViewName]), + exit(Pid, kill), + exit(missing_named_view); +extract_view(Pid, ViewName, [View|Rest], ViewType) -> + case lists:member(ViewName, view_names(View, ViewType)) of + true -> + if ViewType == reduce -> + {index_of(ViewName, view_names(View, reduce)), View}; + true -> + View + end; + false -> + extract_view(Pid, ViewName, Rest, ViewType) + end. + +view_names(View, Type) when Type == red_map; Type == reduce -> + [Name || {Name, _} <- View#view.reduce_funs]; +view_names(View, map) -> + View#view.map_names. + +index_of(X, List) -> + index_of(X, List, 1). + +index_of(_X, [], _I) -> + not_found; +index_of(X, [X|_Rest], I) -> + I; +index_of(X, [_|Rest], I) -> + index_of(X, Rest, I+1). diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl index 6ec7dfde..8316979f 100644 --- a/src/fabric_view_map.erl +++ b/src/fabric_view_map.erl @@ -20,7 +20,7 @@ go(DbName, {GroupId, View}, Args, Callback, Acc0) -> skip = Skip, limit = Limit, stop_fun = stop_fun(Args), - keydict = fabric_view:keydict(Keys), + keys = fabric_view:keydict(Keys), user_acc = Acc0 }, try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, @@ -93,7 +93,7 @@ handle_message(#view_row{} = Row, {Worker, From}, State) -> query_args = #view_query_args{direction=Dir}, counters = Counters0, rows = Rows0, - keydict = KeyDict + keys = KeyDict } = State, Rows = merge_row(Dir, KeyDict, Row#view_row{worker=Worker}, Rows0), Counters1 = fabric_dict:update_counter(Worker, 1, Counters0), diff --git a/src/fabric_view_reduce.erl b/src/fabric_view_reduce.erl new file mode 100644 index 00000000..1a619877 --- /dev/null +++ b/src/fabric_view_reduce.erl @@ -0,0 +1,90 @@ +-module(fabric_view_reduce). + +-export([go/5]). + +-include("fabric.hrl"). + +go(DbName, {GroupId, VName}, Args, Callback, Acc0) -> + {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), + #group{def_lang=Lang, views=Views} = Group = + couch_view_group:design_doc_to_view_group(#db{name=DbName}, DDoc), + {NthRed, View} = fabric_view:extract_view(nil, VName, Views, reduce), + {VName, RedSrc} = lists:nth(NthRed, View#view.reduce_funs), + Workers = lists:map(fun(#shard{name=Name, node=N} = Shard) -> + Ref = rexi:cast(N, {fabric_rpc, reduce_view, [Name,Group,VName,Args]}), + Shard#shard{ref = Ref} + end, partitions:all_parts(DbName)), + BufferSize = couch_config:get("fabric", "reduce_buffer_size", "20"), + #view_query_args{limit = Limit, skip = Skip} = Args, + State = #collector{ + query_args = Args, + callback = Callback, + buffer_size = list_to_integer(BufferSize), + counters = fabric_dict:init(Workers, 0), + skip = Skip, + limit = Limit, + lang = Group#group.def_lang, + os_proc = couch_query_servers:get_os_process(Lang), + reducer = RedSrc, + rows = dict:new(), + user_acc = Acc0 + }, + try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, + State, infinity, 5000) of + {ok, NewState} -> + {ok, NewState#collector.user_acc}; + Error -> + Error + after + fabric_util:cleanup(Workers), + catch couch_query_servers:ret_os_process(State#collector.os_proc) + end. + +handle_message({rexi_DOWN, _, _, _}, nil, State) -> + % TODO see if progress can be made here, possibly by removing all shards + % from that node and checking is_progress_possible + {ok, State}; + +handle_message({rexi_EXIT, Reason}, Worker, State) -> + ?LOG_ERROR("~p rexi_EXIT ~p", [?MODULE, Reason]), + #collector{callback=Callback, counters=Counters0, user_acc=Acc} = State, + Counters = fabric_dict:erase(Worker, Counters0), + case fabric_view:is_progress_possible(Counters) of + true -> + {ok, State#collector{counters = Counters}}; + false -> + Callback({error, dead_shards}, Acc), + {error, dead_shards} + end; + +handle_message(#view_row{key=Key} = Row, {Worker, From}, State) -> + #collector{counters = Counters0, rows = Rows0} = State, + case fabric_dict:lookup_element(Worker, Counters0) of + undefined -> + % this worker lost the race with other partition copies, terminate it + gen_server:reply(From, stop), + {ok, State}; + % first -> + % gen_server:reply(From, ok), + % Rows = dict:append(Key, Row#view_row{worker=Worker}, Rows0), + % C1 = fabric_dict:store(Worker, 1, Counters0), + % C2 = fabric_view:remove_overlapping_shards(Worker, C1), + % NewState = State#collector{counters=C2, rows=Rows}, + % case fabric_dict:any(first, C2) of + % true -> + % {ok, NewState}; + % false -> + % fabric_view:maybe_send_row(State#collector{counters=C2, rows=Rows}) + % end; + _ -> + Rows = dict:append(Key, Row#view_row{worker=Worker}, Rows0), + C1 = fabric_dict:update_counter(Worker, 1, Counters0), + C2 = fabric_view:remove_overlapping_shards(Worker, C1), + State1 = State#collector{rows=Rows, counters=C2}, + State2 = fabric_view:maybe_pause_worker(Worker, From, State1), + fabric_view:maybe_send_row(State2) + end; + +handle_message(complete, Worker, State) -> + Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters), + fabric_view:maybe_send_row(State#collector{counters = Counters}). -- cgit v1.2.3 From 42a45f2a77a125f8c0f5be99edd574fbe391f1ee Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 8 Jun 2010 10:51:08 -0400 Subject: ignore missing Keys in reduce view, like couch. BugzID 10220 --- src/fabric_view.erl | 11 +++++------ src/fabric_view_reduce.erl | 14 ++------------ 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/fabric_view.erl b/src/fabric_view.erl index 70dedf27..2432ab40 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -126,6 +126,8 @@ keydict(Keys) -> %% internal %% +get_next_row(#collector{rows = []}) -> + throw(complete); get_next_row(#collector{reducer = RedSrc} = St) when RedSrc =/= undefined -> #collector{ query_args = #view_query_args{direction=Dir}, @@ -146,11 +148,8 @@ get_next_row(#collector{reducer = RedSrc} = St) when RedSrc =/= undefined -> NewSt = St#collector{keys=RestKeys, rows=NewRowDict, counters=Counters}, {#view_row{key=Key, id=reduced, value=Reduced}, NewSt}; error -> - NewSt = St#collector{keys=RestKeys}, - {#view_row{key=Key, id=reduced, value={error, missing}}, NewSt} + get_next_row(St#collector{keys=RestKeys}) end; -get_next_row(#collector{rows = []}) -> - throw(complete); get_next_row(State) -> #collector{ rows = [Row|Rest], @@ -167,12 +166,12 @@ get_next_row(State) -> {Row, NewState#collector{rows = Rest}} end. -find_next_key(undefined, Dir, RowDict) -> +find_next_key(nil, Dir, RowDict) -> case lists:sort(sort_fun(Dir), dict:fetch_keys(RowDict)) of [] -> throw(complete); [Key|_] -> - {Key, undefined} + {Key, nil} end; find_next_key([], _, _) -> throw(complete); diff --git a/src/fabric_view_reduce.erl b/src/fabric_view_reduce.erl index 1a619877..0e52ec84 100644 --- a/src/fabric_view_reduce.erl +++ b/src/fabric_view_reduce.erl @@ -21,6 +21,7 @@ go(DbName, {GroupId, VName}, Args, Callback, Acc0) -> callback = Callback, buffer_size = list_to_integer(BufferSize), counters = fabric_dict:init(Workers, 0), + keys = Args#view_query_args.keys, skip = Skip, limit = Limit, lang = Group#group.def_lang, @@ -64,21 +65,10 @@ handle_message(#view_row{key=Key} = Row, {Worker, From}, State) -> % this worker lost the race with other partition copies, terminate it gen_server:reply(From, stop), {ok, State}; - % first -> - % gen_server:reply(From, ok), - % Rows = dict:append(Key, Row#view_row{worker=Worker}, Rows0), - % C1 = fabric_dict:store(Worker, 1, Counters0), - % C2 = fabric_view:remove_overlapping_shards(Worker, C1), - % NewState = State#collector{counters=C2, rows=Rows}, - % case fabric_dict:any(first, C2) of - % true -> - % {ok, NewState}; - % false -> - % fabric_view:maybe_send_row(State#collector{counters=C2, rows=Rows}) - % end; _ -> Rows = dict:append(Key, Row#view_row{worker=Worker}, Rows0), C1 = fabric_dict:update_counter(Worker, 1, Counters0), + % TODO time this call, if slow don't do it every time C2 = fabric_view:remove_overlapping_shards(Worker, C1), State1 = State#collector{rows=Rows, counters=C2}, State2 = fabric_view:maybe_pause_worker(Worker, From, State1), -- cgit v1.2.3 From 4e0c97bf3587e9d0e330494f0d06194c0c4bfa17 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 8 Jun 2010 11:39:13 -0400 Subject: allow for unsorted results in map queries, BugzID 10073 --- include/fabric.hrl | 1 + src/fabric_view_map.erl | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/include/fabric.hrl b/include/fabric.hrl index f4665ca8..7fdb5bed 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -26,6 +26,7 @@ os_proc, reducer, lang, + sorted, user_acc }). diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl index 8316979f..b20a0f39 100644 --- a/src/fabric_view_map.erl +++ b/src/fabric_view_map.erl @@ -21,6 +21,7 @@ go(DbName, {GroupId, View}, Args, Callback, Acc0) -> limit = Limit, stop_fun = stop_fun(Args), keys = fabric_view:keydict(Keys), + sorted = Args#view_query_args.sorted, user_acc = Acc0 }, try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, @@ -88,6 +89,17 @@ handle_message({total_and_offset, Tot, Off}, {Worker, From}, State) -> end end; +handle_message(#view_row{}, {_, _}, #collector{limit=0} = State) -> + #collector{callback=Callback} = State, + {_, Acc} = Callback(complete, State#collector.user_acc), + {stop, State#collector{user_acc=Acc}}; + +handle_message(#view_row{} = Row, {_,From}, #collector{sorted=false} = St) -> + #collector{callback=Callback, user_acc=AccIn, limit=Limit} = St, + {Go, Acc} = Callback(fabric_view:transform_row(Row), AccIn), + gen_server:reply(From, ok), + {Go, St#collector{user_acc=Acc, limit=Limit-1}}; + handle_message(#view_row{} = Row, {Worker, From}, State) -> #collector{ query_args = #view_query_args{direction=Dir}, -- cgit v1.2.3 From d7a6bd635ce60d0fc6a6a40dc3027cee308701f6 Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Tue, 8 Jun 2010 12:00:59 -0400 Subject: fabric attachments --- ebin/fabric.app | 1 + src/fabric.erl | 6 ++- src/fabric_doc_attachments.erl | 103 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 src/fabric_doc_attachments.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index ef05bb5d..1fb67200 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -11,6 +11,7 @@ fabric_db_doc_count, fabric_db_info, fabric_dict, + fabric_doc_attachments, fabric_doc_missing_revs, fabric_doc_open, fabric_doc_open_revs, diff --git a/src/fabric.erl b/src/fabric.erl index 983cd818..80bc6d4c 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -6,7 +6,7 @@ % Documents -export([open_doc/3, open_revs/4, get_missing_revs/2, update_doc/3, - update_docs/3]). + update_docs/3, att_receiver/2]). % Views -export([all_docs/4, query_view/5]). @@ -42,7 +42,7 @@ delete_db(DbName, Options) -> fabric_db_delete:delete_db(dbname(DbName), opts(Options)). - +% doc operations open_doc(DbName, Id, Options) -> fabric_doc_open:go(dbname(DbName), docid(Id), opts(Options)). @@ -60,6 +60,8 @@ update_doc(DbName, Doc, Options) -> update_docs(DbName, Docs, Options) -> fabric_doc_update:go(dbname(DbName), docs(Docs), opts(Options)). +att_receiver(Req, Length) -> + fabric_doc_attachments:receiver(Req, Length). all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when is_function(Callback, 2) -> diff --git a/src/fabric_doc_attachments.erl b/src/fabric_doc_attachments.erl new file mode 100644 index 00000000..6230a444 --- /dev/null +++ b/src/fabric_doc_attachments.erl @@ -0,0 +1,103 @@ +-module(fabric_doc_attachments). + +-include("fabric.hrl"). + +%% couch api calls +-export([receiver/2]). + +receiver(_Req, undefined) -> + <<"">>; +receiver(_Req, {unknown_transfer_encoding, Unknown}) -> + exit({unknown_transfer_encoding, Unknown}); +receiver(Req, chunked) -> + % ?LOG_INFO("generating chunked attachment processes", []), + MiddleMan = spawn(fun() -> middleman(Req, chunked) end), + fun(4096, ChunkFun, ok) -> + write_chunks(MiddleMan, ChunkFun) + end; +receiver(_Req, 0) -> + <<"">>; +receiver(Req, Length) when is_integer(Length) -> + Middleman = spawn(fun() -> middleman(Req, Length) end), + fun() -> + Middleman ! {self(), gimme_data}, + receive {Middleman, Data} -> Data end + end; +receiver(_Req, Length) -> + exit({length_not_integer, Length}). + +%% +%% internal +%% + +write_chunks(MiddleMan, ChunkFun) -> + MiddleMan ! {self(), gimme_data}, + receive + {MiddleMan, {0, _Footers}} -> + % MiddleMan ! {self(), done}, + ok; + {MiddleMan, ChunkRecord} -> + ChunkFun(ChunkRecord, ok), + write_chunks(MiddleMan, ChunkFun) + end. + +receive_unchunked_attachment(_Req, 0) -> + ok; +receive_unchunked_attachment(Req, Length) -> + receive {MiddleMan, go} -> + Data = couch_httpd:recv(Req, 0), + MiddleMan ! {self(), Data} + end, + receive_unchunked_attachment(Req, Length - size(Data)). + +middleman(Req, chunked) -> + % spawn a process to actually receive the uploaded data + RcvFun = fun(ChunkRecord, ok) -> + receive {From, go} -> From ! {self(), ChunkRecord} end, ok + end, + Receiver = spawn(fun() -> couch_httpd:recv_chunked(Req,4096,RcvFun,ok) end), + + % take requests from the DB writers and get data from the receiver + N = erlang:list_to_integer(couch_config:get("cluster","n")), + middleman_loop(Receiver, N, dict:new(), 0, []); + +middleman(Req, Length) -> + Receiver = spawn(fun() -> receive_unchunked_attachment(Req, Length) end), + N = erlang:list_to_integer(couch_config:get("cluster","n")), + middleman_loop(Receiver, N, dict:new(), 0, []). + +middleman_loop(Receiver, N, Counters, Offset, ChunkList) -> + receive {From, gimme_data} -> + % figure out how far along this writer (From) is in the list + {NewCounters, WhichChunk} = case dict:find(From, Counters) of + {ok, I} -> + {dict:update_counter(From, 1, Counters), I}; + error -> + {dict:store(From, 2, Counters), 1} + end, + ListIndex = WhichChunk - Offset, + + % talk to the receiver to get another chunk if necessary + ChunkList1 = if ListIndex > length(ChunkList) -> + Receiver ! {self(), go}, + receive {Receiver, ChunkRecord} -> ChunkList ++ [ChunkRecord] end; + true -> ChunkList end, + + % reply to the writer + From ! {self(), lists:nth(ListIndex, ChunkList1)}, + + % check if we can drop a chunk from the head of the list + SmallestIndex = dict:fold(fun(_, Val, Acc) -> lists:min([Val,Acc]) end, + WhichChunk+1, NewCounters), + Size = dict:size(NewCounters), + + {NewChunkList, NewOffset} = + if Size == N andalso (SmallestIndex - Offset) == 2 -> + {tl(ChunkList1), Offset+1}; + true -> + {ChunkList1, Offset} + end, + middleman_loop(Receiver, N, NewCounters, NewOffset, NewChunkList) + after 10000 -> + ok + end. -- cgit v1.2.3 From 44380d7ac0b135fa2d6107cf784a99a1efd393e6 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 8 Jun 2010 16:20:31 -0400 Subject: fabric handler for _changes, BugzID 10219 --- ebin/fabric.app | 1 + src/fabric.erl | 7 +- src/fabric_dict.erl | 5 + src/fabric_rpc.erl | 61 +++++++++-- src/fabric_view_changes.erl | 242 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 src/fabric_view_changes.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index 1fb67200..9c26b635 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -20,6 +20,7 @@ fabric_util, fabric_view, fabric_view_all_docs, + fabric_view_changes, fabric_view_map, fabric_view_reduce ]}, diff --git a/src/fabric.erl b/src/fabric.erl index 80bc6d4c..80646561 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -9,7 +9,7 @@ update_docs/3, att_receiver/2]). % Views --export([all_docs/4, query_view/5]). +-export([all_docs/4, changes/3, query_view/5]). % miscellany -export([db_path/2, design_docs/1]). @@ -67,6 +67,11 @@ all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when is_function(Callback, 2) -> fabric_view_all_docs:go(dbname(DbName), QueryArgs, Callback, Acc0). +changes(DbName, Options, Callback) -> + % TODO use a keylist for Options instead of #changes_args, BugzID 10281 + Feed = Options#changes_args.feed, + fabric_view_changes:go(dbname(DbName), Feed, Options, Callback). + query_view(DbName, View, #view_query_args{view_type=reduce} = QueryArgs, Callback, Acc0) -> fabric_view_reduce:go(dbname(DbName), view(View), QueryArgs, Callback, Acc0); diff --git a/src/fabric_dict.erl b/src/fabric_dict.erl index a4191682..42d46b34 100644 --- a/src/fabric_dict.erl +++ b/src/fabric_dict.erl @@ -11,6 +11,9 @@ init(Keys, InitialValue) -> decrement_all(Dict) -> [{K,V-1} || {K,V} <- Dict]. +store(Key, Value, Dict) -> + orddict:store(Key, Value, Dict). + erase(Key, Dict) -> orddict:erase(Key, Dict). @@ -21,6 +24,8 @@ update_counter(Key, Incr, Dict0) -> lookup_element(Key, Dict) -> couch_util:get_value(Key, Dict). +size(Dict) -> + orddict:size(Dict). any(Value, Dict) -> lists:keymember(Value, 2, Dict). diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index aa922585..fea95f24 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -2,7 +2,7 @@ -export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). --export([all_docs/2, map_view/4, reduce_view/4]). +-export([all_docs/2, changes/3, map_view/4, reduce_view/4]). -include("fabric.hrl"). @@ -42,6 +42,28 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> {ok, Acc} = couch_db:enum_docs(Db, StartId, Dir, fun view_fold/3, Acc0), final_response(Total, Acc#view_acc.offset). +changes(DbName, Args0, StartSeq) -> + #changes_args{style=Style, dir=Dir, filter=FilterName} = Args0, + case couch_db:open(DbName, []) of + {ok, Db} -> + % couch code has a MochiReq for 2nd argument, ick + Args = Args0#changes_args{ + filter = couch_changes:make_filter_fun(FilterName, nil, Db) + }, + Enum = fun changes_enumerator/2, + Opts = [{dir,Dir}], + Acc0 = {Db, StartSeq, Args}, + try + {ok, {_, LastSeq, _}} = + couch_db:changes_since(Db, Style, StartSeq, Enum, Opts, Acc0), + rexi:reply({complete, LastSeq}) + after + couch_db:close(Db) + end; + Error -> + rexi:reply(Error) + end. + map_view(DbName, DDoc, ViewName, QueryArgs) -> {ok, Db} = couch_db:open(DbName, []), #view_query_args{ @@ -124,12 +146,7 @@ get_doc_count(DbName) -> with_db(DbName, [], {couch_db, get_doc_count, []}). get_update_seq(DbName) -> - rexi:reply(case couch_db:open(DbName, []) of - {ok, #db{update_seq = Seq}} -> - {ok, Seq}; - Error -> - Error - end). + with_db(DbName, [], {couch_db, get_update_seq, []}). open_doc(DbName, DocId, Options) -> with_db(DbName, Options, {couch_db, open_doc_int, [DocId, Options]}). @@ -288,3 +305,33 @@ send(Key, Value, #view_acc{limit=Limit} = Acc) -> stop -> exit(normal) end. + +changes_enumerator(DocInfos, {Db, _Seq, Args}) -> + #changes_args{include_docs=IncludeDocs, filter=FilterFun} = Args, + [#doc_info{id=Id, high_seq=Seq, revs=[#rev_info{deleted=Del,rev=Rev}|_]}|_] + = DocInfos, + case [Result || Result <- FilterFun(DocInfos), Result /= null] of + [] -> + {ok, {Db, Seq, Args}}; + Results -> + ChangesRow = changes_row(Db, Seq, Id, Results, Rev, Del, IncludeDocs), + Go = rexi:sync_reply(ChangesRow), + {Go, {Db, Seq, Args}} + end. + +changes_row(_, Seq, Id, Results, _, true, true) -> + #view_row{key=Seq, id=Id, value=Results, doc=deleted}; +changes_row(_, Seq, Id, Results, _, true, false) -> + #view_row{key=Seq, id=Id, value=Results, doc=deleted}; +changes_row(Db, Seq, Id, Results, Rev, false, true) -> + #view_row{key=Seq, id=Id, value=Results, doc=doc_member(Db, Id, Rev)}; +changes_row(_, Seq, Id, Results, _, false, false) -> + #view_row{key=Seq, id=Id, value=Results}. + +doc_member(Shard, Id, Rev) -> + case couch_db:open_doc_revs(Shard, Id, [Rev], []) of + {ok, [{ok,Doc}]} -> + couch_doc:to_json_obj(Doc, []); + Error -> + Error + end. diff --git a/src/fabric_view_changes.erl b/src/fabric_view_changes.erl new file mode 100644 index 00000000..666a85c6 --- /dev/null +++ b/src/fabric_view_changes.erl @@ -0,0 +1,242 @@ +-module(fabric_view_changes). + +-export([go/4, start_update_notifier/1]). + +-include("fabric.hrl"). + +go(DbName, Feed, Options, Callback) when Feed == "continuous" orelse + Feed == "longpoll" -> + Args = make_changes_args(Options), + {ok, Acc0} = Callback(start, Feed), + Notifiers = start_update_notifiers(DbName), + {Timeout, TimeoutFun} = couch_changes:get_changes_timeout(Args, Callback), + try + keep_sending_changes( + DbName, + Args, + Callback, + get_start_seq(DbName, Args), + Acc0, + Timeout, + TimeoutFun + ) + after + stop_update_notifiers(Notifiers), + couch_changes:get_rest_db_updated() + end; + +go(DbName, "normal", Options, Callback) -> + Args = make_changes_args(Options), + {ok, Acc0} = Callback(start, "normal"), + {ok, #collector{counters=Seqs, user_acc=AccOut}} = send_changes( + DbName, + Args, + Callback, + get_start_seq(DbName, Args), + Acc0 + ), + Callback({stop, pack_seqs(Seqs)}, AccOut), + {ok, AccOut}. + +keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, TFun) -> + #changes_args{limit=Limit, feed=Feed} = Args, + {ok, Collector} = send_changes(DbName, Args, Callback, Seqs, AccIn), + #collector{limit=Limit2, counters=Seqs, user_acc=AccOut} = Collector, + LastSeq = pack_seqs(Seqs), + if Limit > Limit2, Feed == "longpoll" -> + Callback({stop, LastSeq}, AccOut); + true -> + case couch_changes:wait_db_updated(Timeout, TFun) of + updated -> + keep_sending_changes( + DbName, + Args#changes_args{limit=Limit2}, + Callback, + LastSeq, + AccIn, + Timeout, + TFun + ); + stop -> + Callback({stop, LastSeq}, AccOut) + end + end. + +send_changes(DbName, ChangesArgs, Callback, PackedSeqs, AccIn) -> + AllShards = partitions:all_parts(DbName), + Seqs = lists:flatmap(fun({#shard{name=Name, node=N} = Shard, Seq}) -> + case lists:member(Shard, AllShards) of + true -> + Ref = rexi:cast(N, {fabric_rpc, changes, [Name,ChangesArgs,Seq]}), + [{Shard#shard{ref = Ref}, Seq}]; + false -> + % Find some replacement shards to cover the missing range + % TODO It's possible in rare cases of shard merging to end up + % with overlapping shard ranges from this technique + lists:map(fun(#shard{name=Name2, node=N2} = NewShard) -> + Ref = rexi:cast(N2, {fabric_rpc, changes, [Name2,ChangesArgs,0]}), + {NewShard#shard{ref = Ref}, 0} + end, find_replacement_shards(Shard, AllShards)) + end + end, unpack_seqs(PackedSeqs, DbName)), + {Workers, _} = lists:unzip(Seqs), + State = #collector{ + query_args = ChangesArgs, + callback = Callback, + counters = fabric_dict:init(Workers, 0), + user_acc = AccIn, + limit = ChangesArgs#changes_args.limit, + rows = Seqs % store sequence positions instead + }, + try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, + State, infinity, 5000) + after + fabric_util:cleanup(Workers) + end. + +handle_message({rexi_DOWN, _, _, _}, nil, State) -> + % TODO see if progress can be made here, possibly by removing all shards + % from that node and checking is_progress_possible + {ok, State}; + +handle_message({rexi_EXIT, Reason}, Worker, State) -> + ?LOG_ERROR("~p rexi_EXIT ~p", [?MODULE, Reason]), + #collector{ + callback=Callback, + counters=Counters0, + rows = Seqs0, + user_acc=Acc + } = State, + Counters = fabric_dict:erase(Worker, Counters0), + Seqs = fabric_dict:erase(Worker, Seqs0), + case fabric_view:is_progress_possible(Counters) of + true -> + {ok, State#collector{counters = Counters, rows=Seqs}}; + false -> + Callback({error, dead_shards}, Acc), + {error, dead_shards} + end; + +handle_message(_, _, #collector{limit=0} = State) -> + {stop, State}; + +handle_message(#view_row{key=Seq} = Row0, {Worker, From}, St) -> + #collector{ + query_args = #changes_args{include_docs=IncludeDocs}, + callback = Callback, + counters = S0, + limit = Limit, + user_acc = AccIn + } = St, + case fabric_dict:lookup_element(Worker, S0) of + undefined -> + % this worker lost the race with other partition copies, terminate it + gen_server:reply(From, stop), + {ok, St}; + _ -> + S1 = fabric_dict:store(Worker, Seq, S0), + S2 = fabric_view:remove_overlapping_shards(Worker, S1), + Row = Row0#view_row{key = pack_seqs(S2)}, + {Go, Acc} = Callback(changes_row(Row, IncludeDocs), AccIn), + gen_server:reply(From, Go), + {Go, St#collector{counters=S2, limit=Limit-1, user_acc=Acc}} + end; + +handle_message({complete, EndSeq}, Worker, State) -> + #collector{ + counters = S0, + total_rows = Completed % override + } = State, + case fabric_dict:lookup_element(Worker, S0) of + undefined -> + {ok, State}; + _ -> + S1 = fabric_dict:store(Worker, EndSeq, S0), + % unlikely to have overlaps here, but possible w/ filters + S2 = fabric_view:remove_overlapping_shards(Worker, S1), + NewState = State#collector{counters=S2, total_rows=Completed+1}, + case fabric_dict:size(S2) =:= (Completed+1) of + true -> + {stop, NewState}; + false -> + {ok, NewState} + end + end. + +make_changes_args(Options) -> + Options. + +get_start_seq(_DbName, #changes_args{dir=fwd, since=Since}) -> + Since; +get_start_seq(DbName, #changes_args{dir=rev}) -> + Shards = partitions:all_parts(DbName), + Workers = fabric_util:submit_jobs(Shards, get_update_seq, []), + {ok, Since} = fabric_util:recv(Workers, #shard.ref, + fun collect_update_seqs/3, fabric_dict:init(Workers, -1)), + Since. + +collect_update_seqs(Seq, Shard, Counters) when is_integer(Seq) -> + case fabric_dict:lookup_element(Shard, Counters) of + undefined -> + % already heard from someone else in this range + {ok, Counters}; + -1 -> + C1 = fabric_dict:store(Shard, Seq, Counters), + C2 = fabric_view:remove_overlapping_shards(Shard, C1), + case fabric_dict:any(-1, C2) of + true -> + {ok, C2}; + false -> + {stop, pack_seqs(C2)} + end + end. + +pack_seqs(Workers) -> + SeqList = [{N,R,S} || {#shard{node=N, range=R}, S} <- Workers], + couch_util:encodeBase64Url(term_to_binary(SeqList, [compressed])). + +unpack_seqs(0, DbName) -> + fabric_dict:init(partitions:all_parts(DbName), 0); + +unpack_seqs("0", DbName) -> + fabric_dict:init(partitions:all_parts(DbName), 0); + +unpack_seqs(Packed, DbName) -> + % TODO relies on internal structure of fabric_dict as keylist + lists:map(fun({Node, [A,B], Seq}) -> + Name = partitions:shard_name(DbName, A), + {#shard{node=Node, range=[A,B], dbname=DbName, name=Name}, Seq} + end, binary_to_term(couch_util:decodeBase64Url(Packed))). + +start_update_notifiers(DbName) -> + lists:map(fun(#shard{node=Node, name=Name}) -> + {Node, rexi:cast(Node, {?MODULE, start_update_notifier, [Name]})} + end, partitions:all_parts(DbName)). + +% rexi endpoint +start_update_notifier(DbName) -> + {Caller, _} = get(rexi_from), + Fun = fun({_, X}) when X == DbName -> Caller ! db_updated; (_) -> ok end, + Id = {couch_db_update_notifier, make_ref()}, + ok = gen_event:add_sup_handler(couch_db_update, Id, Fun), + receive {gen_event_EXIT, Id, Reason} -> + rexi:reply({gen_event_EXIT, DbName, Reason}) + end. + +stop_update_notifiers(Notifiers) -> + [rexi:kill(Node, Ref) || {Node, Ref} <- Notifiers]. + +changes_row(#view_row{key=Seq, id=Id, value=Value, doc=deleted}, true) -> + {change, {[{seq,Seq}, {id,Id}, {changes,Value}, {deleted, true}, {doc, null}]}}; +changes_row(#view_row{key=Seq, id=Id, value=Value, doc=deleted}, false) -> + {change, {[{seq,Seq}, {id,Id}, {changes,Value}, {deleted, true}]}}; +changes_row(#view_row{key=Seq, id=Id, value=Value, doc={error,Reason}}, true) -> + {change, {[{seq,Seq}, {id,Id}, {changes,Value}, {error,Reason}]}}; +changes_row(#view_row{key=Seq, id=Id, value=Value, doc=Doc}, true) -> + {change, {[{seq,Seq}, {id,Id}, {changes,Value}, {doc,Doc}]}}; +changes_row(#view_row{key=Seq, id=Id, value=Value}, false) -> + {change, {[{seq,Seq}, {id,Id}, {changes,Value}]}}. + +find_replacement_shards(#shard{range=Range}, AllShards) -> + % TODO make this moar betta -- we might have split or merged the partition + [Shard || Shard <- AllShards, Shard#shard.range =:= Range]. -- cgit v1.2.3 From e8c43b1e8e6f5393e579ae9e015c5b7c282dbf2a Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 9 Jun 2010 14:02:21 -0400 Subject: rewrite fabric_db_info --- src/fabric.erl | 6 +-- src/fabric_db_info.erl | 124 +++++++++++++++++-------------------------------- 2 files changed, 45 insertions(+), 85 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index 80646561..e4887953 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,7 +1,7 @@ -module(fabric). % DBs --export([all_dbs/0, all_dbs/1, create_db/2, delete_db/2, get_db_info/2, +-export([all_dbs/0, all_dbs/1, create_db/2, delete_db/2, get_db_info/1, get_doc_count/1]). % Documents @@ -29,8 +29,8 @@ all_dbs() -> all_dbs(Customer) -> fabric_all_databases:all_databases(Customer). -get_db_info(DbName, Customer) -> - fabric_db_info:get_db_info(dbname(DbName), Customer). +get_db_info(DbName) -> + fabric_db_info:go(dbname(DbName)). get_doc_count(DbName) -> fabric_db_doc_count:go(dbname(DbName)). diff --git a/src/fabric_db_info.erl b/src/fabric_db_info.erl index e70b335c..8db7212f 100644 --- a/src/fabric_db_info.erl +++ b/src/fabric_db_info.erl @@ -1,91 +1,51 @@ -module(fabric_db_info). --author(brad@cloudant.com). --export([get_db_info/2]). +-export([go/1]). -include("fabric.hrl"). -%% @doc get database information tuple -get_db_info(DbName, Customer) -> - Name = cloudant_db_name(Customer, DbName), +go(DbName) -> Shards = partitions:all_parts(DbName), Workers = fabric_util:submit_jobs(Shards, get_db_info, []), - Acc0 = {false, length(Workers), lists:usort([ {Beg, nil} || - #shard{range=[Beg,_]} <- Workers])}, - case fabric_util:recv(Workers, #shard.ref, fun handle_info_msg/3, Acc0) of - {ok, ShardInfos} -> - {ok, process_infos(ShardInfos, [{db_name, Name}])}; - Error -> Error - end. - - -%% ===================== -%% internal -%% ===================== - -handle_info_msg(_, _, {true, _, Infos0}) -> - {stop, Infos0}; -handle_info_msg(_, _, {false, 1, Infos0}) -> - MissingShards = lists:reverse(lists:foldl(fun - ({S,nil}, Acc) -> [S|Acc]; - (_, Acc) -> Acc - end, [], Infos0)), - ?LOG_ERROR("get_db_info error, missing shards: ~p", [MissingShards]), - {error, get_db_info}; -handle_info_msg({ok, Info}, #shard{range=[Beg,_]}, {false, N, Infos0}) -> - case couch_util:get_value(Beg, Infos0) of + Acc0 = {fabric_dict:init(Workers, nil), []}, + fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). + +handle_message({ok, Info}, #shard{dbname=Name} = Shard, {Counters, Acc}) -> + case fabric_dict:lookup_element(Shard, Counters) of + undefined -> + % already heard from someone else in this range + {ok, {Counters, Acc}}; nil -> - Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Info}), - case is_complete(Infos) of - true -> {ok, {true, N-1, Infos}}; - false -> {ok, {false, N-1, Infos}} - end; - _ -> - {ok, {false, N-1, Infos0}} + C1 = fabric_dict:store(Shard, ok, Counters), + C2 = fabric_view:remove_overlapping_shards(Shard, C1), + case fabric_dict:any(nil, C2) of + true -> + {ok, {C2, [Info|Acc]}}; + false -> + {stop, [{db_name,Name}|merge_results(lists:flatten([Info|Acc]))]} + end end; -handle_info_msg(_Other, _, {Complete, N, Infos0}) -> - {ok, {Complete, N-1, Infos0}}. - -is_complete(List) -> - not lists:any(fun({_,Info}) -> Info =:= nil end, List). - -cloudant_db_name(Customer, FullName) -> - case Customer of - "" -> FullName; - Name -> re:replace(FullName, [Name,"/"], "", [{return, binary}]) - end. - -%% Loop through Tasks on the flattened Infos and get the aggregated result -process_infos(Infos, Initial) -> - Tasks = [ - {doc_count, fun sum/2, 0}, - {doc_del_count, fun sum/2, 0}, - {update_seq, fun max/2, 1}, - {purge_seq, fun sum/2, 0}, - {compact_running, fun bool/2, 0}, - {disk_size, fun sum/2, 0}, - {instance_start_time, fun(_, _) -> <<"0">> end, 0}, - {disk_format_version, fun max/2, 0}], - - Infos1 = lists:flatten(Infos), - - Result = lists:map(fun({Type, Fun, Default}) -> - {Type, process_info(Type, Fun, Default, Infos1)} - end, Tasks), - lists:flatten([Initial, Result]). - - process_info(Type, Fun, Default, List) -> - lists:foldl(fun(V, AccIn) -> Fun(V, AccIn) end, Default, - proplists:get_all_values(Type, List)). - -sum(New, Existing) -> - New + Existing. - -bool(New, Existing) -> - New andalso Existing. - -max(New, Existing) -> - case New > Existing of - true -> New; - false -> Existing - end. +handle_message(_, _, Acc) -> + {ok, Acc}. + +merge_results(Info) -> + Dict = lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end, + orddict:new(), Info), + orddict:fold(fun + (doc_count, X, Acc) -> + [{doc_count, lists:sum(X)} | Acc]; + (doc_del_count, X, Acc) -> + [{doc_del_count, lists:sum(X)} | Acc]; + (update_seq, X, Acc) -> + [{update_seq, lists:sum(X)} | Acc]; + (purge_seq, X, Acc) -> + [{purge_seq, lists:sum(X)} | Acc]; + (compact_running, X, Acc) -> + [{compact_running, lists:member(true, X)} | Acc]; + (disk_size, X, Acc) -> + [{disk_size, lists:sum(X)} | Acc]; + (disk_format_version, X, Acc) -> + [{disk_format_version, lists:max(X)} | Acc]; + (_, _, Acc) -> + Acc + end, [{instance_start_time, <<"0">>}], Dict). -- cgit v1.2.3 From a4cd71dec2ca37cfca9a7ac15b3ff31f47cd11d7 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 9 Jun 2010 14:56:42 -0400 Subject: some ?COUCH cleanup, and replicated_changes updates --- src/fabric.erl | 40 ++++++++++++++++++++++++++++++---------- src/fabric_rpc.erl | 8 +++++++- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index e4887953..80c9614e 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -9,7 +9,7 @@ update_docs/3, att_receiver/2]). % Views --export([all_docs/4, changes/3, query_view/5]). +-export([all_docs/4, changes/3, query_view/3, query_view/4, query_view/6]). % miscellany -export([db_path/2, design_docs/1]). @@ -72,11 +72,22 @@ changes(DbName, Options, Callback) -> Feed = Options#changes_args.feed, fabric_view_changes:go(dbname(DbName), Feed, Options, Callback). -query_view(DbName, View, #view_query_args{view_type=reduce} = QueryArgs, - Callback, Acc0) -> - fabric_view_reduce:go(dbname(DbName), view(View), QueryArgs, Callback, Acc0); -query_view(DbName, View, #view_query_args{} = QueryArgs, Callback, Acc0) -> - fabric_view_map:go(dbname(DbName), view(View), QueryArgs, Callback, Acc0). +query_view(DbName, DesignName, ViewName) -> + query_view(DbName, DesignName, ViewName, #view_query_args{}). + +query_view(DbName, DesignName, ViewName, QueryArgs) -> + Callback = fun default_callback/2, + query_view(DbName, DesignName, ViewName, QueryArgs, Callback, []). + +query_view(DbName, DesignName, ViewName, QueryArgs, Callback, Acc0) -> + Db = dbname(DbName), Design = name(DesignName), View = name(ViewName), + case is_reduce_view(Db, Design, View, QueryArgs) of + true -> + Mod = fabric_view_reduce; + false -> + Mod = fabric_view_map + end, + Mod:go(Db, Design, View, QueryArgs, Callback, Acc0). design_docs(DbName) -> QueryArgs = #view_query_args{start_key = <<"_design/">>, include_docs=true}, @@ -100,9 +111,14 @@ dbname(DbName) when is_list(DbName) -> list_to_binary(DbName); dbname(DbName) when is_binary(DbName) -> DbName; +dbname(#db{name=Name}) -> + Name; dbname(DbName) -> erlang:error({illegal_database_name, DbName}). +name(Thing) -> + couch_util:to_binary(Thing). + docid(DocId) when is_list(DocId) -> list_to_binary(DocId); docid(DocId) when is_binary(DocId) -> @@ -130,10 +146,6 @@ rev(Rev) when is_list(Rev); is_binary(Rev) -> rev({Seq, Hash} = Rev) when is_integer(Seq), is_binary(Hash) -> Rev. -view(ViewName) -> - [Group, View] = re:split(ViewName, "/"), - {Group, View}. - opts(Options) -> case couch_util:get_value(user_ctx, Options) of undefined -> @@ -147,6 +159,14 @@ opts(Options) -> Options end. +default_callback(complete, Acc) -> + {ok, lists:reverse(Acc)}; +default_callback(Row, Acc) -> + {ok, [Row | Acc]}. + +is_reduce_view(_, _, _, #view_query_args{view_type=Reduce}) -> + Reduce =:= reduce. + generate_customer_path("/", _Customer) -> ""; generate_customer_path("/favicon.ico", _Customer) -> diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index fea95f24..f1f84ed5 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -172,7 +172,13 @@ get_missing_revs(DbName, IdRevsList) -> end). update_docs(DbName, Docs, Options) -> - with_db(DbName, Options, {couch_db, update_docs, [Docs, Options]}). + case proplists:get_value(replicated_changes, Options) of + true -> + X = replicated_changes; + _ -> + X = interactive_edit + end, + with_db(DbName, Options, {couch_db, update_docs, [Docs, Options, X]}). %% %% internal -- cgit v1.2.3 From 00e161c34b6fc71c09ffe376d6c8b9512814216a Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 9 Jun 2010 15:36:58 -0400 Subject: bunch o' updates - fabric view group info - send user_ctx to fabric on update_docs requests - send 403s for compaction and view cleanup --- ebin/fabric.app | 1 + src/fabric.erl | 6 +++++- src/fabric_rpc.erl | 6 +++++- src/fabric_view_map.erl | 4 ++-- src/fabric_view_reduce.erl | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index 9c26b635..b93387a4 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -16,6 +16,7 @@ fabric_doc_open, fabric_doc_open_revs, fabric_doc_update, + fabric_group_info, fabric_rpc, fabric_util, fabric_view, diff --git a/src/fabric.erl b/src/fabric.erl index 80c9614e..c3a858ce 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -9,7 +9,8 @@ update_docs/3, att_receiver/2]). % Views --export([all_docs/4, changes/3, query_view/3, query_view/4, query_view/6]). +-export([all_docs/4, changes/3, query_view/3, query_view/4, query_view/6, + get_view_group_info/2]). % miscellany -export([db_path/2, design_docs/1]). @@ -89,6 +90,9 @@ query_view(DbName, DesignName, ViewName, QueryArgs, Callback, Acc0) -> end, Mod:go(Db, Design, View, QueryArgs, Callback, Acc0). +get_view_group_info(DbName, DesignId) -> + fabric_group_info:go(dbname(DbName), name(DesignId)). + design_docs(DbName) -> QueryArgs = #view_query_args{start_key = <<"_design/">>, include_docs=true}, Callback = fun({total_and_offset, _, _}, []) -> diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index f1f84ed5..49e61370 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -2,7 +2,7 @@ -export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). --export([all_docs/2, changes/3, map_view/4, reduce_view/4]). +-export([all_docs/2, changes/3, map_view/4, reduce_view/4, group_info/2]). -include("fabric.hrl"). @@ -180,6 +180,10 @@ update_docs(DbName, Docs, Options) -> end, with_db(DbName, Options, {couch_db, update_docs, [Docs, Options, X]}). +group_info(DbName, Group0) -> + {ok, Pid} = gen_server:call(couch_view, {get_group_server, DbName, Group0}), + rexi:reply(couch_view_group:request_group_info(Pid)). + %% %% internal %% diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl index b20a0f39..e152c2ed 100644 --- a/src/fabric_view_map.erl +++ b/src/fabric_view_map.erl @@ -1,10 +1,10 @@ -module(fabric_view_map). --export([go/5]). +-export([go/6]). -include("fabric.hrl"). -go(DbName, {GroupId, View}, Args, Callback, Acc0) -> +go(DbName, GroupId, View, Args, Callback, Acc0) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> Ref = rexi:cast(Node, {fabric_rpc, map_view, [Name, DDoc, View, Args]}), diff --git a/src/fabric_view_reduce.erl b/src/fabric_view_reduce.erl index 0e52ec84..af92f98e 100644 --- a/src/fabric_view_reduce.erl +++ b/src/fabric_view_reduce.erl @@ -4,7 +4,7 @@ -include("fabric.hrl"). -go(DbName, {GroupId, VName}, Args, Callback, Acc0) -> +go(DbName, GroupId, VName, Args, Callback, Acc0) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), #group{def_lang=Lang, views=Views} = Group = couch_view_group:design_doc_to_view_group(#db{name=DbName}, DDoc), -- cgit v1.2.3 From 34467d1f47d55b6b793f338b200eecf935b81cd4 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 9 Jun 2010 16:33:33 -0400 Subject: _all_docs served over http via fabric, woot --- src/fabric_view_reduce.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fabric_view_reduce.erl b/src/fabric_view_reduce.erl index af92f98e..73395d8c 100644 --- a/src/fabric_view_reduce.erl +++ b/src/fabric_view_reduce.erl @@ -1,6 +1,6 @@ -module(fabric_view_reduce). --export([go/5]). +-export([go/6]). -include("fabric.hrl"). -- cgit v1.2.3 From 5b4205315fd9b1c0745974705c67c68e0cb1c8a7 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 9 Jun 2010 17:07:05 -0400 Subject: updates to .app resource files --- ebin/fabric.app | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index b93387a4..ffd5f82f 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -1,8 +1,8 @@ %% fabric app resource file {application, fabric, - [{description, "clustered couchdb functions"}, - {vsn, "0.1.0"}, + [{description, "Routing and proxying layer for CouchDB cluster"}, + {vsn, "1.0"}, {modules, [ fabric, fabric_all_databases, @@ -25,8 +25,5 @@ fabric_view_map, fabric_view_reduce ]}, - {registered, []}, - {included_applications, []}, - {applications, [kernel, stdlib, couch, rexi, membership]}, - {start_phases, []} + {applications, [kernel, stdlib, couch, rexi, membership]} ]}. -- cgit v1.2.3 From a8fc23d74b3ddfc44bacbd1c82389e2150234ff7 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 9 Jun 2010 21:18:47 -0400 Subject: map and reduce views working over HTTP now, too --- src/fabric.erl | 4 ++-- src/fabric_view_map.erl | 5 ++++- src/fabric_view_reduce.erl | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index c3a858ce..dcd751a9 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -80,8 +80,8 @@ query_view(DbName, DesignName, ViewName, QueryArgs) -> Callback = fun default_callback/2, query_view(DbName, DesignName, ViewName, QueryArgs, Callback, []). -query_view(DbName, DesignName, ViewName, QueryArgs, Callback, Acc0) -> - Db = dbname(DbName), Design = name(DesignName), View = name(ViewName), +query_view(DbName, Design, ViewName, QueryArgs, Callback, Acc0) -> + Db = dbname(DbName), View = name(ViewName), case is_reduce_view(Db, Design, View, QueryArgs) of true -> Mod = fabric_view_reduce; diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl index e152c2ed..c314586c 100644 --- a/src/fabric_view_map.erl +++ b/src/fabric_view_map.erl @@ -4,8 +4,11 @@ -include("fabric.hrl"). -go(DbName, GroupId, View, Args, Callback, Acc0) -> +go(DbName, GroupId, View, Args, Callback, Acc0) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), + go(DbName, DDoc, View, Args, Callback, Acc0); + +go(DbName, DDoc, View, Args, Callback, Acc0) -> Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> Ref = rexi:cast(Node, {fabric_rpc, map_view, [Name, DDoc, View, Args]}), Shard#shard{ref = Ref} diff --git a/src/fabric_view_reduce.erl b/src/fabric_view_reduce.erl index 73395d8c..9514bdef 100644 --- a/src/fabric_view_reduce.erl +++ b/src/fabric_view_reduce.erl @@ -4,8 +4,11 @@ -include("fabric.hrl"). -go(DbName, GroupId, VName, Args, Callback, Acc0) -> +go(DbName, GroupId, View, Args, Callback, Acc0) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), + go(DbName, DDoc, View, Args, Callback, Acc0); + +go(DbName, DDoc, VName, Args, Callback, Acc0) -> #group{def_lang=Lang, views=Views} = Group = couch_view_group:design_doc_to_view_group(#db{name=DbName}, DDoc), {NthRed, View} = fabric_view:extract_view(nil, VName, Views, reduce), -- cgit v1.2.3 From e089037a7761b2ded9846bfa30929c364f4af504 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 10 Jun 2010 14:16:26 -0400 Subject: add handler for view group _info --- src/fabric_group_info.erl | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/fabric_group_info.erl diff --git a/src/fabric_group_info.erl b/src/fabric_group_info.erl new file mode 100644 index 00000000..ec716378 --- /dev/null +++ b/src/fabric_group_info.erl @@ -0,0 +1,48 @@ +-module(fabric_group_info). + +-export([go/2]). + +-include("fabric.hrl"). + +go(DbName, GroupId) -> + {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), + Group = couch_view_group:design_doc_to_view_group(#db{name=DbName}, DDoc), + Shards = partitions:all_parts(DbName), + Workers = fabric_util:submit_jobs(Shards, group_info, [Group]), + Acc0 = {fabric_dict:init(Workers, nil), []}, + fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). + +handle_message({ok, Info}, Shard, {Counters, Acc}) -> + case fabric_dict:lookup_element(Shard, Counters) of + undefined -> + % already heard from someone else in this range + {ok, {Counters, Acc}}; + nil -> + C1 = fabric_dict:store(Shard, ok, Counters), + C2 = fabric_view:remove_overlapping_shards(Shard, C1), + case fabric_dict:any(nil, C2) of + true -> + {ok, {C2, [Info|Acc]}}; + false -> + {stop, merge_results(lists:flatten([Info|Acc]))} + end + end; +handle_message(M, _, Acc) -> + ?LOG_INFO("mismatch ~p", [M]), + {ok, Acc}. + +merge_results(Info) -> + Dict = lists:foldl(fun({K,V},D0) -> orddict:append(K,V,D0) end, + orddict:new(), Info), + orddict:fold(fun + (signature, [X|_], Acc) -> + [{signature, X} | Acc]; + (language, [X|_], Acc) -> + [{language, X} | Acc]; + (disk_size, X, Acc) -> + [{disk_size, lists:sum(X)} | Acc]; + (compact_running, X, Acc) -> + [{compact_running, lists:member(true, X)} | Acc]; + (_, _, Acc) -> + Acc + end, [], Dict). -- cgit v1.2.3 From 2e3dd3c20a669f5e9b21e49877b8731e2c4afc50 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 10 Jun 2010 15:18:07 -0400 Subject: removing some logging --- src/fabric_doc_attachments.erl | 1 - src/fabric_group_info.erl | 1 - src/fabric_util.erl | 1 - 3 files changed, 3 deletions(-) diff --git a/src/fabric_doc_attachments.erl b/src/fabric_doc_attachments.erl index 6230a444..aecdaaef 100644 --- a/src/fabric_doc_attachments.erl +++ b/src/fabric_doc_attachments.erl @@ -10,7 +10,6 @@ receiver(_Req, undefined) -> receiver(_Req, {unknown_transfer_encoding, Unknown}) -> exit({unknown_transfer_encoding, Unknown}); receiver(Req, chunked) -> - % ?LOG_INFO("generating chunked attachment processes", []), MiddleMan = spawn(fun() -> middleman(Req, chunked) end), fun(4096, ChunkFun, ok) -> write_chunks(MiddleMan, ChunkFun) diff --git a/src/fabric_group_info.erl b/src/fabric_group_info.erl index ec716378..bb860837 100644 --- a/src/fabric_group_info.erl +++ b/src/fabric_group_info.erl @@ -28,7 +28,6 @@ handle_message({ok, Info}, Shard, {Counters, Acc}) -> end end; handle_message(M, _, Acc) -> - ?LOG_INFO("mismatch ~p", [M]), {ok, Acc}. merge_results(Info) -> diff --git a/src/fabric_util.erl b/src/fabric_util.erl index f0ab213d..f89e9404 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -6,7 +6,6 @@ submit_jobs(Shards, EndPoint, ExtraArgs) -> lists:map(fun(#shard{node=Node, name=ShardName} = Shard) -> - io:format("submitting ~p ~p~n", [Node, {fabric_rpc, EndPoint, [ShardName | ExtraArgs]}]), Ref = rexi:cast(Node, {fabric_rpc, EndPoint, [ShardName | ExtraArgs]}), Shard#shard{ref = Ref} end, Shards). -- cgit v1.2.3 From c8644425d8f9f5a1f247952301f1305c1a043701 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 10 Jun 2010 21:39:05 -0400 Subject: 10 seconds is much too short for e.g. _bulk_docs timeout --- src/fabric_util.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fabric_util.erl b/src/fabric_util.erl index f89e9404..90b0c647 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -17,7 +17,7 @@ recv(Workers, Keypos, Fun, Acc0) -> receive_loop(Workers, Keypos, Fun, Acc0). receive_loop(Workers, Keypos, Fun, Acc0) -> - case couch_config:get("fabric", "request_timeout", "10000") of + case couch_config:get("fabric", "request_timeout", "60000") of "infinity" -> Timeout = infinity; N -> -- cgit v1.2.3 From 699dda57fc02239a7b0f1740b5b82ba3763a66d1 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 11 Jun 2010 14:00:05 -0400 Subject: suppress unused variable warning --- src/fabric_group_info.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fabric_group_info.erl b/src/fabric_group_info.erl index bb860837..42c29838 100644 --- a/src/fabric_group_info.erl +++ b/src/fabric_group_info.erl @@ -27,7 +27,7 @@ handle_message({ok, Info}, Shard, {Counters, Acc}) -> {stop, merge_results(lists:flatten([Info|Acc]))} end end; -handle_message(M, _, Acc) -> +handle_message(_, _, Acc) -> {ok, Acc}. merge_results(Info) -> -- cgit v1.2.3 From cc6785194931a8c1b4e481a47decb1a3afc49e8d Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 11 Jun 2010 16:07:16 -0400 Subject: updates to better support _changes HTTP resource --- src/fabric.erl | 6 +++--- src/fabric_view_changes.erl | 23 +++++++++++------------ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index dcd751a9..5abb17e3 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -9,7 +9,7 @@ update_docs/3, att_receiver/2]). % Views --export([all_docs/4, changes/3, query_view/3, query_view/4, query_view/6, +-export([all_docs/4, changes/4, query_view/3, query_view/4, query_view/6, get_view_group_info/2]). % miscellany @@ -68,10 +68,10 @@ all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when is_function(Callback, 2) -> fabric_view_all_docs:go(dbname(DbName), QueryArgs, Callback, Acc0). -changes(DbName, Options, Callback) -> +changes(DbName, Options, Callback, Acc0) -> % TODO use a keylist for Options instead of #changes_args, BugzID 10281 Feed = Options#changes_args.feed, - fabric_view_changes:go(dbname(DbName), Feed, Options, Callback). + fabric_view_changes:go(dbname(DbName), Feed, Options, Callback, Acc0). query_view(DbName, DesignName, ViewName) -> query_view(DbName, DesignName, ViewName, #view_query_args{}). diff --git a/src/fabric_view_changes.erl b/src/fabric_view_changes.erl index 666a85c6..39a57176 100644 --- a/src/fabric_view_changes.erl +++ b/src/fabric_view_changes.erl @@ -1,13 +1,13 @@ -module(fabric_view_changes). --export([go/4, start_update_notifier/1]). +-export([go/5, start_update_notifier/1]). -include("fabric.hrl"). -go(DbName, Feed, Options, Callback) when Feed == "continuous" orelse +go(DbName, Feed, Options, Callback, Acc0) when Feed == "continuous" orelse Feed == "longpoll" -> Args = make_changes_args(Options), - {ok, Acc0} = Callback(start, Feed), + {ok, Acc} = Callback(start, Acc0), Notifiers = start_update_notifiers(DbName), {Timeout, TimeoutFun} = couch_changes:get_changes_timeout(Args, Callback), try @@ -16,7 +16,7 @@ go(DbName, Feed, Options, Callback) when Feed == "continuous" orelse Args, Callback, get_start_seq(DbName, Args), - Acc0, + Acc, Timeout, TimeoutFun ) @@ -25,24 +25,23 @@ go(DbName, Feed, Options, Callback) when Feed == "continuous" orelse couch_changes:get_rest_db_updated() end; -go(DbName, "normal", Options, Callback) -> +go(DbName, "normal", Options, Callback, Acc0) -> Args = make_changes_args(Options), - {ok, Acc0} = Callback(start, "normal"), + {ok, Acc} = Callback(start, Acc0), {ok, #collector{counters=Seqs, user_acc=AccOut}} = send_changes( DbName, Args, Callback, get_start_seq(DbName, Args), - Acc0 + Acc ), - Callback({stop, pack_seqs(Seqs)}, AccOut), - {ok, AccOut}. + Callback({stop, pack_seqs(Seqs)}, AccOut). keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, TFun) -> #changes_args{limit=Limit, feed=Feed} = Args, {ok, Collector} = send_changes(DbName, Args, Callback, Seqs, AccIn), - #collector{limit=Limit2, counters=Seqs, user_acc=AccOut} = Collector, - LastSeq = pack_seqs(Seqs), + #collector{limit=Limit2, counters=NewSeqs, user_acc=AccOut} = Collector, + LastSeq = pack_seqs(NewSeqs), if Limit > Limit2, Feed == "longpoll" -> Callback({stop, LastSeq}, AccOut); true -> @@ -204,7 +203,7 @@ unpack_seqs("0", DbName) -> unpack_seqs(Packed, DbName) -> % TODO relies on internal structure of fabric_dict as keylist lists:map(fun({Node, [A,B], Seq}) -> - Name = partitions:shard_name(DbName, A), + Name = partitions:shard_name(A, DbName), {#shard{node=Node, range=[A,B], dbname=DbName, name=Name}, Seq} end, binary_to_term(couch_util:decodeBase64Url(Packed))). -- cgit v1.2.3 From ec60d8d7f8dc709fd0ef0f33c26d7834af32f96a Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 14 Jun 2010 17:16:56 -0400 Subject: update fabric to use couch 0.11 btree api --- src/fabric_rpc.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 49e61370..51cc31a3 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -39,7 +39,8 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> total_rows = Total, stop_fun = all_docs_stop_fun(QueryArgs) }, - {ok, Acc} = couch_db:enum_docs(Db, StartId, Dir, fun view_fold/3, Acc0), + Options = [{start_key, StartId}, {dir, Dir}], + {ok, Acc} = couch_db:enum_docs(Db, fun view_fold/3, Acc0, Options), final_response(Total, Acc#view_acc.offset). changes(DbName, Args0, StartSeq) -> -- cgit v1.2.3 From 68b5a1ffe4913601e7469e3e1017d663d4d06ea8 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 14 Jun 2010 17:17:43 -0400 Subject: minor updates to _changes code --- src/fabric_rpc.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 51cc31a3..da4a9b8c 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -44,12 +44,12 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> final_response(Total, Acc#view_acc.offset). changes(DbName, Args0, StartSeq) -> - #changes_args{style=Style, dir=Dir, filter=FilterName} = Args0, + #changes_args{style=Style, dir=Dir} = Args0, case couch_db:open(DbName, []) of {ok, Db} -> % couch code has a MochiReq for 2nd argument, ick Args = Args0#changes_args{ - filter = couch_changes:make_filter_fun(FilterName, nil, Db) + filter = couch_changes:make_filter_fun(Args0, nil, Db) }, Enum = fun changes_enumerator/2, Opts = [{dir,Dir}], -- cgit v1.2.3 From 2bdd685dc2c426fc74f5d2c89d8e34653a1c55aa Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 14 Jun 2010 18:57:09 -0400 Subject: update all_docs and map rpc endpoints for 0.11 btree --- src/fabric_rpc.erl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index da4a9b8c..d0558697 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -40,7 +40,7 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> stop_fun = all_docs_stop_fun(QueryArgs) }, Options = [{start_key, StartId}, {dir, Dir}], - {ok, Acc} = couch_db:enum_docs(Db, fun view_fold/3, Acc0, Options), + {ok, _, Acc} = couch_db:enum_docs(Db, fun view_fold/3, Acc0, Options), final_response(Total, Acc#view_acc.offset). changes(DbName, Args0, StartSeq) -> @@ -95,14 +95,16 @@ map_view(DbName, DDoc, ViewName, QueryArgs) -> }, case Keys of nil -> - {ok, Acc} = couch_view:fold(View, Start, Dir, fun view_fold/3, Acc0); + Options = [{start_key, Start}, {dir, Dir}], + {ok, _, Acc} = couch_view:fold(View, fun view_fold/3, Acc0, Options); _ -> Acc = lists:foldl(fun(Key, AccIn) -> KeyStart = {Key, StartDocId}, KeyStop = default_stop_fun(QueryArgs#view_query_args{start_key=Key, end_key=Key}), - {_Go, Out} = couch_view:fold(View, KeyStart, Dir, fun view_fold/3, - AccIn#view_acc{stop_fun = KeyStop}), + Options = [{start_key, KeyStart}, {dir, Dir}], + {_Go, _, Out} = couch_view:fold(View, fun view_fold/3, + AccIn#view_acc{stop_fun = KeyStop}, Options), Out end, Acc0, Keys) end, -- cgit v1.2.3 From 9692364437c47ef9e13607e996112839f4f48876 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 15 Jun 2010 11:05:37 -0400 Subject: update reduce views to work with 0.11 btree api --- src/fabric_rpc.erl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index d0558697..26aebfb5 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -132,13 +132,17 @@ reduce_view(DbName, Group0, ViewName, QueryArgs) -> {NthRed, View} = fabric_view:extract_view(Pid, ViewName, Views, reduce), ReduceView = {reduce, NthRed, Lang, View}, Acc0 = #view_acc{group_level = GroupLevel, limit = Limit+Skip}, + Options0 = [{key_group_fun, GroupFun}, {dir, Dir}], case Keys of nil -> - couch_view:fold_reduce(ReduceView, Dir, {StartKey,StartDocId}, - {EndKey,EndDocId}, GroupFun, fun reduce_fold/3, Acc0); + Options = [{start_key, {StartKey,StartDocId}}, + {end_key, {EndKey,EndDocId}} | Options0], + couch_view:fold_reduce(ReduceView, fun reduce_fold/3, Acc0, Options); _ -> - [couch_view:fold_reduce(ReduceView, Dir, {K,StartDocId}, {K,EndDocId}, - GroupFun, fun reduce_fold/3, Acc0) || K <- Keys] + + [couch_view:fold_reduce(ReduceView, fun reduce_fold/3, Acc0, + [{start_key,{K,StartDocId}}, {end_key,{K,EndDocId}} | Options0]) + || K <- Keys] end, rexi:reply(complete). -- cgit v1.2.3 From 691908d1fa774ea8f90f44f0185d40c75a60c1f2 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 15 Jun 2010 11:57:46 -0400 Subject: throw conflicts just like couch_db.erl --- src/fabric.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/fabric.erl b/src/fabric.erl index 5abb17e3..afa87082 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -56,7 +56,12 @@ get_missing_revs(DbName, IdsRevs) when is_list(IdsRevs) -> update_doc(DbName, Doc, Options) -> {ok, [Result]} = update_docs(DbName, [Doc], opts(Options)), - Result. + case Result of + {ok, _} -> + Result; + Error -> + throw(Error) + end. update_docs(DbName, Docs, Options) -> fabric_doc_update:go(dbname(DbName), docs(Docs), opts(Options)). -- cgit v1.2.3 From 340f2240aa05f1d198610649a599210c87450863 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 15 Jun 2010 13:38:44 -0400 Subject: updates to work w/ 0.11 btree option refactoring --- include/fabric.hrl | 1 - src/fabric_rpc.erl | 122 ++++++++++++++----------------------------- src/fabric_view.erl | 13 +---- src/fabric_view_all_docs.erl | 10 ---- src/fabric_view_map.erl | 27 ---------- 5 files changed, 40 insertions(+), 133 deletions(-) diff --git a/include/fabric.hrl b/include/fabric.hrl index 7fdb5bed..68516ffb 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -21,7 +21,6 @@ rows = [], skip, limit, - stop_fun, keys, os_proc, reducer, diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 26aebfb5..c9bfb7d2 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -13,7 +13,6 @@ offset = nil, total_rows, reduce_fun = fun couch_db:enum_docs_reduce_to_count/1, - stop_fun, group_level = 0 }). @@ -25,21 +24,27 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> #view_query_args{ start_key = StartKey, start_docid = StartDocId, + end_key = EndKey, + end_docid = EndDocId, limit = Limit, skip = Skip, include_docs = IncludeDocs, - direction = Dir + direction = Dir, + inclusive_end = Inclusive } = QueryArgs, - StartId = if is_binary(StartKey) -> StartKey; true -> StartDocId end, {ok, Total} = couch_db:get_doc_count(Db), Acc0 = #view_acc{ db = Db, include_docs = IncludeDocs, limit = Limit+Skip, - total_rows = Total, - stop_fun = all_docs_stop_fun(QueryArgs) + total_rows = Total }, - Options = [{start_key, StartId}, {dir, Dir}], + EndKeyType = if Inclusive -> end_key; true -> end_key_gt end, + Options = [ + {dir, Dir}, + {start_key, if is_binary(StartKey) -> StartKey; true -> StartDocId end}, + {EndKeyType, if is_binary(EndKey) -> EndKey; true -> EndDocId end} + ], {ok, _, Acc} = couch_db:enum_docs(Db, fun view_fold/3, Acc0, Options), final_response(Total, Acc#view_acc.offset). @@ -68,17 +73,13 @@ changes(DbName, Args0, StartSeq) -> map_view(DbName, DDoc, ViewName, QueryArgs) -> {ok, Db} = couch_db:open(DbName, []), #view_query_args{ - start_key = StartKey, - start_docid = StartDocId, limit = Limit, skip = Skip, keys = Keys, include_docs = IncludeDocs, - direction = Dir, stale = Stale, view_type = ViewType } = QueryArgs, - Start = {StartKey, StartDocId}, MinSeq = if Stale == ok -> 0; true -> couch_db:get_update_seq(Db) end, Group0 = couch_view_group:design_doc_to_view_group(Db, DDoc), {ok, Pid} = gen_server:call(couch_view, {get_group_server, DbName, Group0}), @@ -90,21 +91,18 @@ map_view(DbName, DDoc, ViewName, QueryArgs) -> include_docs = IncludeDocs, limit = Limit+Skip, total_rows = Total, - reduce_fun = fun couch_view:reduce_to_count/1, - stop_fun = default_stop_fun(QueryArgs) + reduce_fun = fun couch_view:reduce_to_count/1 }, case Keys of nil -> - Options = [{start_key, Start}, {dir, Dir}], + Options = couch_httpd_view:make_key_options(QueryArgs), {ok, _, Acc} = couch_view:fold(View, fun view_fold/3, Acc0, Options); _ -> Acc = lists:foldl(fun(Key, AccIn) -> - KeyStart = {Key, StartDocId}, - KeyStop = default_stop_fun(QueryArgs#view_query_args{start_key=Key, - end_key=Key}), - Options = [{start_key, KeyStart}, {dir, Dir}], - {_Go, _, Out} = couch_view:fold(View, fun view_fold/3, - AccIn#view_acc{stop_fun = KeyStop}, Options), + KeyArgs = QueryArgs#view_query_args{start_key=Key, end_key=Key}, + Options = couch_httpd_view:make_key_options(KeyArgs), + {_Go, _, Out} = couch_view:fold(View, fun view_fold/3, AccIn, + Options), Out end, Acc0, Keys) end, @@ -113,10 +111,6 @@ map_view(DbName, DDoc, ViewName, QueryArgs) -> reduce_view(DbName, Group0, ViewName, QueryArgs) -> {ok, Db} = couch_db:open(DbName, []), #view_query_args{ - start_key = StartKey, - start_docid = StartDocId, - end_key = EndKey, - end_docid = EndDocId, group_level = GroupLevel, limit = Limit, skip = Skip, @@ -135,14 +129,16 @@ reduce_view(DbName, Group0, ViewName, QueryArgs) -> Options0 = [{key_group_fun, GroupFun}, {dir, Dir}], case Keys of nil -> - Options = [{start_key, {StartKey,StartDocId}}, - {end_key, {EndKey,EndDocId}} | Options0], + Options0 = couch_httpd_view:make_key_options(QueryArgs), + Options = [{key_group_fun, GroupFun} | Options0], couch_view:fold_reduce(ReduceView, fun reduce_fold/3, Acc0, Options); _ -> - - [couch_view:fold_reduce(ReduceView, fun reduce_fold/3, Acc0, - [{start_key,{K,StartDocId}}, {end_key,{K,EndDocId}} | Options0]) - || K <- Keys] + lists:map(fun(Key) -> + KeyArgs = QueryArgs#view_query_args{start_key=Key, end_key=Key}, + Options0 = couch_httpd_view:make_key_options(KeyArgs), + Options = [{key_group_fun, GroupFun} | Options0], + couch_view:fold_reduce(ReduceView, fun reduce_fold/3, Acc0, Options) + end, Keys) end, rexi:reply(complete). @@ -231,35 +227,20 @@ view_fold({{Key,Id}, Value}, _Offset, Acc) -> #view_acc{ db = Db, limit = Limit, - include_docs = IncludeDocs, - stop_fun = PassedEnd + include_docs = IncludeDocs } = Acc, - case PassedEnd(Key, Id) of - true -> - {stop, Acc}; - false -> - Doc = if not IncludeDocs -> undefined; true -> - case couch_db:open_doc(Db, Id, []) of - {not_found, deleted} -> - null; - {not_found, missing} -> - undefined; - {ok, Doc0} -> - couch_doc:to_json_obj(Doc0, []) - end - end, - rexi:sync_reply(#view_row{key=Key, id=Id, value=Value, doc=Doc}), - {ok, Acc#view_acc{limit=Limit-1}} - end. - -all_docs_stop_fun(#view_query_args{direction=fwd, end_key=EndKey}) -> - fun(ViewKey, _) -> - couch_db_updater:less_docid(EndKey, ViewKey) - end; -all_docs_stop_fun(#view_query_args{direction=rev, end_key=EndKey}) -> - fun(ViewKey, _) -> - couch_db_updater:less_docid(ViewKey, EndKey) - end. + Doc = if not IncludeDocs -> undefined; true -> + case couch_db:open_doc(Db, Id, []) of + {not_found, deleted} -> + null; + {not_found, missing} -> + undefined; + {ok, Doc0} -> + couch_doc:to_json_obj(Doc0, []) + end + end, + rexi:sync_reply(#view_row{key=Key, id=Id, value=Value, doc=Doc}), + {ok, Acc#view_acc{limit=Limit-1}}. final_response(Total, nil) -> case rexi:sync_reply({total_and_offset, Total, Total}) of ok -> @@ -268,33 +249,6 @@ final_response(Total, nil) -> final_response(_Total, _Offset) -> rexi:reply(complete). -default_stop_fun(#view_query_args{direction=fwd, inclusive_end=true} = Args) -> - #view_query_args{end_key=EndKey, end_docid=EndDocId} = Args, - fun(ViewKey, ViewId) -> - couch_view:less_json([EndKey, EndDocId], [ViewKey, ViewId]) - end; -default_stop_fun(#view_query_args{direction=fwd} = Args) -> - #view_query_args{end_key=EndKey, end_docid=EndDocId} = Args, - fun - (ViewKey, _ViewId) when ViewKey == EndKey -> - true; - (ViewKey, ViewId) -> - couch_view:less_json([EndKey, EndDocId], [ViewKey, ViewId]) - end; -default_stop_fun(#view_query_args{direction=rev, inclusive_end=true} = Args) -> - #view_query_args{end_key=EndKey, end_docid=EndDocId} = Args, - fun(ViewKey, ViewId) -> - couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId]) - end; -default_stop_fun(#view_query_args{direction=rev} = Args) -> - #view_query_args{end_key=EndKey, end_docid=EndDocId} = Args, - fun - (ViewKey, _ViewId) when ViewKey == EndKey -> - true; - (ViewKey, ViewId) -> - couch_view:less_json([ViewKey, ViewId], [EndKey, EndDocId]) - end. - group_rows_fun(exact) -> fun({Key1,_}, {Key2,_}) -> Key1 == Key2 end; group_rows_fun(0) -> diff --git a/src/fabric_view.erl b/src/fabric_view.erl index 2432ab40..99cf1671 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -151,20 +151,11 @@ get_next_row(#collector{reducer = RedSrc} = St) when RedSrc =/= undefined -> get_next_row(St#collector{keys=RestKeys}) end; get_next_row(State) -> - #collector{ - rows = [Row|Rest], - counters = Counters0, - stop_fun = Stop - } = State, + #collector{rows = [Row|Rest], counters = Counters0} = State, Worker = Row#view_row.worker, Counters1 = fabric_dict:update_counter(Worker, -1, Counters0), NewState = maybe_resume_worker(Worker, State#collector{counters=Counters1}), - case Stop(Row) of - true -> - throw(complete); - false -> - {Row, NewState#collector{rows = Rest}} - end. + {Row, NewState#collector{rows = Rest}}. find_next_key(nil, Dir, RowDict) -> case lists:sort(sort_fun(Dir), dict:fetch_keys(RowDict)) of diff --git a/src/fabric_view_all_docs.erl b/src/fabric_view_all_docs.erl index 99834286..196d6837 100644 --- a/src/fabric_view_all_docs.erl +++ b/src/fabric_view_all_docs.erl @@ -19,7 +19,6 @@ go(DbName, #view_query_args{keys=nil} = QueryArgs, Callback, Acc0) -> counters = fabric_dict:init(Workers, 0), skip = Skip, limit = Limit, - stop_fun = stop_fun(QueryArgs), user_acc = Acc0 }, try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, @@ -120,15 +119,6 @@ handle_message(complete, Worker, State) -> fabric_view:maybe_send_row(State#collector{counters = Counters}). -stop_fun(#view_query_args{direction=fwd, end_key=EndKey}) -> - fun(#view_row{id=Id}) -> - couch_db_updater:less_docid(EndKey, Id) - end; -stop_fun(#view_query_args{direction=rev, end_key=EndKey}) -> - fun(#view_row{id=Id}) -> - couch_db_updater:less_docid(Id, EndKey) - end. - merge_row(fwd, Row, Rows) -> lists:keymerge(#view_row.id, [Row], Rows); merge_row(rev, Row, Rows) -> diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl index c314586c..b686b5fc 100644 --- a/src/fabric_view_map.erl +++ b/src/fabric_view_map.erl @@ -22,7 +22,6 @@ go(DbName, DDoc, View, Args, Callback, Acc0) -> counters = fabric_dict:init(Workers, 0), skip = Skip, limit = Limit, - stop_fun = stop_fun(Args), keys = fabric_view:keydict(Keys), sorted = Args#view_query_args.sorted, user_acc = Acc0 @@ -120,32 +119,6 @@ handle_message(complete, Worker, State) -> Counters = fabric_dict:update_counter(Worker, 1, State#collector.counters), fabric_view:maybe_send_row(State#collector{counters = Counters}). -stop_fun(#view_query_args{} = QueryArgs) -> - #view_query_args{ - direction = Dir, - inclusive_end = Inclusive, - end_key = EndKey, - end_docid = EndDocId - } = QueryArgs, - stop_fun(Dir, Inclusive, EndKey, EndDocId). - -stop_fun(fwd, true, EndKey, EndDocId) -> - fun(#view_row{key=Key, id=Id}) -> - couch_view:less_json([EndKey, EndDocId], [Key, Id]) - end; -stop_fun(fwd, false, EndKey, EndDocId) -> - fun(#view_row{key=K}) when K==EndKey -> true; (#view_row{key=Key, id=Id}) -> - couch_view:less_json([EndKey, EndDocId], [Key, Id]) - end; -stop_fun(rev, true, EndKey, EndDocId) -> - fun(#view_row{key=Key, id=Id}) -> - couch_view:less_json([Key, Id], [EndKey, EndDocId]) - end; -stop_fun(rev, false, EndKey, EndDocId) -> - fun(#view_row{key=K}) when K==EndKey -> true; (#view_row{key=Key, id=Id}) -> - couch_view:less_json([Key, Id], [EndKey, EndDocId]) - end. - merge_row(fwd, undefined, Row, Rows) -> lists:merge(fun(#view_row{key=KeyA, id=IdA}, #view_row{key=KeyB, id=IdB}) -> couch_view:less_json([KeyA, IdA], [KeyB, IdB]) -- cgit v1.2.3 From eda54b6051020c6ab1812b6676338a6bad7f467d Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 15 Jun 2010 14:52:28 -0400 Subject: remove leftover code from reduce_view --- src/fabric_rpc.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index c9bfb7d2..e9d9eb20 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -115,7 +115,6 @@ reduce_view(DbName, Group0, ViewName, QueryArgs) -> limit = Limit, skip = Skip, keys = Keys, - direction = Dir, stale = Stale } = QueryArgs, GroupFun = group_rows_fun(GroupLevel), @@ -126,7 +125,6 @@ reduce_view(DbName, Group0, ViewName, QueryArgs) -> {NthRed, View} = fabric_view:extract_view(Pid, ViewName, Views, reduce), ReduceView = {reduce, NthRed, Lang, View}, Acc0 = #view_acc{group_level = GroupLevel, limit = Limit+Skip}, - Options0 = [{key_group_fun, GroupFun}, {dir, Dir}], case Keys of nil -> Options0 = couch_httpd_view:make_key_options(QueryArgs), -- cgit v1.2.3 From 7d91ca78fb286cf216b91229b196c21dd5e192af Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 16 Jun 2010 16:45:44 -0400 Subject: don't hang when deleting a deleted DB --- src/fabric_db_delete.erl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/fabric_db_delete.erl b/src/fabric_db_delete.erl index 0803400d..95b1a5ef 100644 --- a/src/fabric_db_delete.erl +++ b/src/fabric_db_delete.erl @@ -11,7 +11,7 @@ delete_db(DbName, Options) -> Fullmap = partitions:all_parts(DbName), RefPartMap = send_delete_calls(Fullmap, Options), - Acc0 = {true, length(RefPartMap)}, + Acc0 = {not_found, length(RefPartMap)}, case fabric_util:receive_loop( RefPartMap, 1, fun handle_delete_msg/3, Acc0) of {ok, _Results} -> @@ -33,19 +33,19 @@ send_delete_calls(Parts, Options) -> {Ref, Part} end, Parts). -handle_delete_msg(not_found, _, {NotFound, N}) -> - {ok, {NotFound, N-1}}; -handle_delete_msg({rexi_EXIT, _Reason}, _, {NotFound, N}) -> - {ok, {NotFound, N-1}}; +handle_delete_msg(ok, _, {_, 1}) -> + {stop, ok}; +handle_delete_msg(not_found, _, {Acc, 1}) -> + {stop, Acc}; +handle_delete_msg({rexi_EXIT, _Reason}, _, {Acc, N}) -> + % TODO is this the appropriate action to take, or should we abort? + {ok, {Acc, N-1}}; handle_delete_msg({rexi_DOWN, _, _, _}, _, _Acc) -> {error, delete_db_fubar}; -handle_delete_msg(_, _, {NotFound, 1}) -> - if - NotFound -> {stop, not_found}; - true -> {stop, ok} - end; -handle_delete_msg(ok, _, {_NotFound, N}) -> - {ok, {false, N-1}}. +handle_delete_msg(not_found, _, {Acc, N}) -> + {ok, {Acc, N-1}}; +handle_delete_msg(ok, _, {_, N}) -> + {ok, {ok, N-1}}. delete_fullmap(DbName) -> case couch_db:open(<<"dbs">>, []) of -- cgit v1.2.3 From e6fe7126c88893dd39cec32481b05f7c38e97475 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 17 Jun 2010 11:48:06 -0400 Subject: correct handling of PUTs to missing DB --- src/fabric_doc_update.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index 555b1897..4d8ca3a6 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -43,7 +43,11 @@ handle_message({ok, Replies}, Worker, Acc0) -> {_, N} when N < DocCount -> % no point in trying to finalize anything yet {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}} - end. + end; +handle_message({not_found, no_db_file} = X, Worker, Acc0) -> + {_, _, _, GroupedDocs, _} = Acc0, + Docs = couch_util:get_value(Worker, GroupedDocs), + handle_message({ok, [X || _D <- Docs]}, Worker, Acc0). force_reply(Doc, Replies, {W, Acc}) -> % TODO make a real decision here -- cgit v1.2.3 From b7d46723fc704eb37555ac5dce96eab1724bdc2e Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 17 Jun 2010 12:34:11 -0400 Subject: updates to use 0.11-style missing_revs --- src/fabric_doc_missing_revs.erl | 2 +- src/fabric_rpc.erl | 25 ++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/fabric_doc_missing_revs.erl b/src/fabric_doc_missing_revs.erl index fe6deac6..8f1f5b4c 100644 --- a/src/fabric_doc_missing_revs.erl +++ b/src/fabric_doc_missing_revs.erl @@ -55,7 +55,7 @@ group_idrevs_by_shard(DbName, IdsRevs) -> end, dict:new(), IdsRevs)). update_dict(D0, KVs) -> - lists:foldl(fun({K,V}, D1) -> dict:store(K, V, D1) end, D0, KVs). + lists:foldl(fun({K,V,_}, D1) -> dict:store(K, V, D1) end, D0, KVs). skip_message({1, Dict}) -> {stop, dict:fold(fun force_reply/3, [], Dict)}; diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index e9d9eb20..4b6e4e62 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -162,10 +162,11 @@ get_missing_revs(DbName, IdRevsList) -> Ids = [Id1 || {Id1, _Revs} <- IdRevsList], {ok, lists:zipwith(fun({Id, Revs}, FullDocInfoResult) -> case FullDocInfoResult of - {ok, #full_doc_info{rev_tree=RevisionTree}} -> - {Id, couch_key_tree:find_missing(RevisionTree, Revs)}; + {ok, #full_doc_info{rev_tree=RevisionTree} = FullInfo} -> + MissingRevs = couch_key_tree:find_missing(RevisionTree, Revs), + {Id, MissingRevs, possible_ancestors(FullInfo, MissingRevs)}; not_found -> - {Id, Revs} + {Id, Revs, []} end end, IdRevsList, couch_btree:lookup(Db#db.id_tree, Ids))}; Error -> @@ -304,3 +305,21 @@ doc_member(Shard, Id, Rev) -> Error -> Error end. + +possible_ancestors(_FullInfo, []) -> + []; +possible_ancestors(FullInfo, MissingRevs) -> + #doc_info{revs=RevsInfo} = couch_doc:to_doc_info(FullInfo), + LeafRevs = [Rev || #rev_info{rev=Rev} <- RevsInfo], + % Find the revs that are possible parents of this rev + lists:foldl(fun({LeafPos, LeafRevId}, Acc) -> + % this leaf is a "possible ancenstor" of the missing + % revs if this LeafPos lessthan any of the missing revs + case lists:any(fun({MissingPos, _}) -> + LeafPos < MissingPos end, MissingRevs) of + true -> + [{LeafPos, LeafRevId} | Acc]; + false -> + Acc + end + end, [], LeafRevs). -- cgit v1.2.3 From 096343af965c9f371c74e7fd9bd45bbe67f3470f Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 22 Jun 2010 08:05:44 -0400 Subject: allow a #doc{} instead of docid in get_view_group_info --- src/fabric.erl | 11 ++++++++++- src/fabric_group_info.erl | 7 +++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index afa87082..badc1379 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -96,7 +96,7 @@ query_view(DbName, Design, ViewName, QueryArgs, Callback, Acc0) -> Mod:go(Db, Design, View, QueryArgs, Callback, Acc0). get_view_group_info(DbName, DesignId) -> - fabric_group_info:go(dbname(DbName), name(DesignId)). + fabric_group_info:go(dbname(DbName), design_doc(DesignId)). design_docs(DbName) -> QueryArgs = #view_query_args{start_key = <<"_design/">>, include_docs=true}, @@ -147,6 +147,15 @@ doc({_} = Doc) -> doc(Doc) -> erlang:error({illegal_doc_format, Doc}). +design_doc(#doc{} = DDoc) -> + DDoc; +design_doc(DocId) when is_list(DocId) -> + design_doc(list_to_binary(DocId)); +design_doc(<<"_design/", _/binary>> = DocId) -> + DocId; +design_doc(GroupName) -> + <<"_design/", GroupName/binary>>. + idrevs({Id, Revs}) when is_list(Revs) -> {docid(Id), [rev(R) || R <- Revs]}. diff --git a/src/fabric_group_info.erl b/src/fabric_group_info.erl index 42c29838..d2b76674 100644 --- a/src/fabric_group_info.erl +++ b/src/fabric_group_info.erl @@ -4,8 +4,11 @@ -include("fabric.hrl"). -go(DbName, GroupId) -> - {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), +go(DbName, GroupId) when is_binary(GroupId) -> + {ok, DDoc} = fabric:open_doc(DbName, GroupId, []), + go(DbName, DDoc); + +go(DbName, #doc{} = DDoc) -> Group = couch_view_group:design_doc_to_view_group(#db{name=DbName}, DDoc), Shards = partitions:all_parts(DbName), Workers = fabric_util:submit_jobs(Shards, group_info, [Group]), -- cgit v1.2.3 From 6dcb12c20d0dc06e4a536674da41cde77ea3965f Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 24 Jun 2010 17:38:42 -0700 Subject: trivially reorder json props to pass list_views test --- src/fabric_view.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fabric_view.erl b/src/fabric_view.erl index 99cf1671..738bb7dd 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -174,11 +174,11 @@ transform_row(#view_row{key=Key, id=reduced, value=Value}) -> transform_row(#view_row{key=Key, id=undefined}) -> {row, {[{key,Key}, {error,not_found}]}}; transform_row(#view_row{key=Key, id=Id, value=Value, doc=undefined}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}]}}; + {row, {[{id,Id}, {key,Key}, {value,Value}]}}; transform_row(#view_row{key=Key, id=Id, value=Value, doc={error,Reason}}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}, {error,Reason}]}}; + {row, {[{id,Id}, {key,Key}, {value,Value}, {error,Reason}]}}; transform_row(#view_row{key=Key, id=Id, value=Value, doc=Doc}) -> - {row, {[{key,Key}, {id,Id}, {value,Value}, {doc,Doc}]}}. + {row, {[{id,Id}, {key,Key}, {value,Value}, {doc,Doc}]}}. sort_fun(fwd) -> fun(A,A) -> true; (A,B) -> couch_view:less_json(A,B) end; -- cgit v1.2.3 From 5f23313f8424fc5d8f54b2550c0d3a290de21c7f Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 25 Jun 2010 20:52:17 -0400 Subject: filter should already be set outside of fabric --- src/fabric_rpc.erl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 4b6e4e62..04aebfd1 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -48,14 +48,10 @@ all_docs(DbName, #view_query_args{keys=nil} = QueryArgs) -> {ok, _, Acc} = couch_db:enum_docs(Db, fun view_fold/3, Acc0, Options), final_response(Total, Acc#view_acc.offset). -changes(DbName, Args0, StartSeq) -> - #changes_args{style=Style, dir=Dir} = Args0, +changes(DbName, Args, StartSeq) -> + #changes_args{style=Style, dir=Dir} = Args, case couch_db:open(DbName, []) of {ok, Db} -> - % couch code has a MochiReq for 2nd argument, ick - Args = Args0#changes_args{ - filter = couch_changes:make_filter_fun(Args0, nil, Db) - }, Enum = fun changes_enumerator/2, Opts = [{dir,Dir}], Acc0 = {Db, StartSeq, Args}, -- cgit v1.2.3 From 170ad0b78691b1060c0a5ba8eb7fbf29482f6bfe Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 28 Jun 2010 14:18:44 -0400 Subject: wait 1 hour for view requests --- src/fabric_view_map.erl | 2 +- src/fabric_view_reduce.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl index b686b5fc..69133f3b 100644 --- a/src/fabric_view_map.erl +++ b/src/fabric_view_map.erl @@ -27,7 +27,7 @@ go(DbName, DDoc, View, Args, Callback, Acc0) -> user_acc = Acc0 }, try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, - State, infinity, 5000) of + State, infinity, 1000 * 60 * 60) of {ok, NewState} -> {ok, NewState#collector.user_acc}; Error -> diff --git a/src/fabric_view_reduce.erl b/src/fabric_view_reduce.erl index 9514bdef..ca137314 100644 --- a/src/fabric_view_reduce.erl +++ b/src/fabric_view_reduce.erl @@ -34,7 +34,7 @@ go(DbName, DDoc, VName, Args, Callback, Acc0) -> user_acc = Acc0 }, try fabric_util:receive_loop(Workers, #shard.ref, fun handle_message/3, - State, infinity, 5000) of + State, infinity, 1000 * 60 * 60) of {ok, NewState} -> {ok, NewState#collector.user_acc}; Error -> -- cgit v1.2.3 From c79affde7e4bdc0b76dae6b05f35c3a63d7dfc00 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 28 Jun 2010 14:19:17 -0400 Subject: replicated_changes updates are noreply --- src/fabric_doc_update.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index 4d8ca3a6..9d3cd3d1 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -11,10 +11,11 @@ go(DbName, AllDocs, Options) -> end, group_docs_by_shard(DbName, AllDocs)), {Workers, _} = lists:unzip(GroupedDocs), Acc0 = {length(Workers), length(AllDocs), couch_util:get_value(w, Options, 1), - GroupedDocs, dict:new()}, + GroupedDocs, dict:from_list([{Doc,[]} || Doc <- AllDocs])}, case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of {ok, Results} -> - {ok, couch_util:reorder_results(AllDocs, Results)}; + Reordered = couch_util:reorder_results(AllDocs, Results), + {ok, [R || R <- Reordered, R =/= noreply]}; Else -> Else end. @@ -88,6 +89,9 @@ group_docs_by_shard(DbName, Docs) -> append_update_replies([], [], DocReplyDict) -> DocReplyDict; +append_update_replies([Doc|Rest], [], Dict0) -> + % icky, if replicated_changes only errors show up in result + append_update_replies(Rest, [], dict:append(Doc, noreply, Dict0)); append_update_replies([Doc|Rest1], [Reply|Rest2], Dict0) -> % TODO what if the same document shows up twice in one update_docs call? append_update_replies(Rest1, Rest2, dict:append(Doc, Reply, Dict0)). -- cgit v1.2.3 From 796f0c7fca7c17628dcf42fa0ce8c91fd10c4100 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 30 Jun 2010 16:46:36 -0400 Subject: trivial update to includes for mem3 rename --- include/fabric.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fabric.hrl b/include/fabric.hrl index 68516ffb..5cb337d9 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -5,7 +5,7 @@ -endif. -ifndef(MEMBERSHIP). --include("../../membership/include/membership.hrl"). +-include("../../mem3/include/mem3.hrl"). -endif. -include_lib("eunit/include/eunit.hrl"). -- cgit v1.2.3 From 2436c4fba43d02bd0a73893895b43e198715082b Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 1 Jul 2010 10:10:04 -0400 Subject: more renaming so make dist works --- ebin/fabric.app | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index ffd5f82f..6750766b 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -25,5 +25,5 @@ fabric_view_map, fabric_view_reduce ]}, - {applications, [kernel, stdlib, couch, rexi, membership]} + {applications, [kernel, stdlib, couch, rexi, mem3]} ]}. -- cgit v1.2.3 From ced7cd5afff2f79b91f364f713d9851038e3b6ab Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 2 Jul 2010 03:02:12 -0400 Subject: update for mem3 refactor, more robust DB create/delete --- src/fabric.erl | 14 ++++-- src/fabric_db_create.erl | 102 ++++++++++++++++++---------------------- src/fabric_db_delete.erl | 78 ++++++++++++------------------ src/fabric_db_doc_count.erl | 2 +- src/fabric_db_info.erl | 2 +- src/fabric_doc_missing_revs.erl | 2 +- src/fabric_doc_open.erl | 2 +- src/fabric_doc_open_revs.erl | 2 +- src/fabric_doc_update.erl | 2 +- src/fabric_group_info.erl | 2 +- src/fabric_rpc.erl | 14 ++++++ src/fabric_view.erl | 44 +++++++++-------- src/fabric_view_all_docs.erl | 2 +- src/fabric_view_changes.erl | 10 ++-- src/fabric_view_map.erl | 2 +- src/fabric_view_reduce.erl | 2 +- 16 files changed, 138 insertions(+), 144 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index badc1379..b233677b 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,8 +1,8 @@ -module(fabric). % DBs --export([all_dbs/0, all_dbs/1, create_db/2, delete_db/2, get_db_info/1, - get_doc_count/1]). +-export([all_dbs/0, all_dbs/1, create_db/1, create_db/2, delete_db/1, + delete_db/2, get_db_info/1, get_doc_count/1]). % Documents -export([open_doc/3, open_revs/4, get_missing_revs/2, update_doc/3, @@ -36,11 +36,17 @@ get_db_info(DbName) -> get_doc_count(DbName) -> fabric_db_doc_count:go(dbname(DbName)). +create_db(DbName) -> + create_db(DbName, []). + create_db(DbName, Options) -> - fabric_db_create:create_db(dbname(DbName), opts(Options)). + fabric_db_create:go(dbname(DbName), opts(Options)). + +delete_db(DbName) -> + delete_db(DbName, []). delete_db(DbName, Options) -> - fabric_db_delete:delete_db(dbname(DbName), opts(Options)). + fabric_db_delete:go(dbname(DbName), opts(Options)). % doc operations diff --git a/src/fabric_db_create.erl b/src/fabric_db_create.erl index 4f4e3b20..80bd1eb1 100644 --- a/src/fabric_db_create.erl +++ b/src/fabric_db_create.erl @@ -1,64 +1,56 @@ -module(fabric_db_create). --author(brad@cloudant.com). - --export([create_db/2]). +-export([go/2]). -include("fabric.hrl"). %% @doc Create a new database, and all its partition files across the cluster %% Options is proplist with user_ctx, n, q --spec create_db(binary(), list()) -> {ok, #db{}} | {error, any()}. -create_db(DbName, Options) -> - Fullmap = partitions:fullmap(DbName, Options), - {ok, FullNodes} = mem3:fullnodes(), - RefPartMap = send_create_calls(Fullmap, Options), - Acc0 = {false, length(RefPartMap), lists:usort([ {Beg, false} || - {_,#shard{range=[Beg,_]}} <- RefPartMap])}, - Result = case fabric_util:receive_loop( - RefPartMap, 1, fun handle_create_msg/3, Acc0) of - {ok, _Results} -> ok; - Error -> Error - end, - % always install partition map, even w/ errors, so delete is possible - partitions:install_fullmap(DbName, Fullmap, FullNodes, Options), - Result. - -%% -%% internal -%% +go(DbName, Options) -> + Shards = mem3:choose_shards(DbName, Options), + Doc = make_document(Shards), + Workers = fabric_util:submit_jobs(Shards, create_db, [Options, Doc]), + Acc0 = fabric_dict:init(Workers, nil), + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, _} -> + ok; + Else -> + Else + end. -%% @doc create the partitions on all appropriate nodes (rexi calls) --spec send_create_calls(fullmap(), list()) -> [{reference(), part()}]. -send_create_calls(Fullmap, Options) -> - lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> - Ref = rexi:async_server_call({couch_server, Node}, - {create, ShardName, Options}), - {Ref, Part} - end, Fullmap). +handle_message(Msg, Shard, Counters) -> + C1 = fabric_dict:store(Shard, Msg, Counters), + case fabric_dict:any(nil, C1) of + true -> + {ok, C1}; + false -> + final_answer(C1) + end. -%% @doc handle create messages from shards -handle_create_msg(file_exists, _, _) -> - {error, file_exists}; -handle_create_msg({rexi_EXIT, _Reason}, _, {Complete, N, Parts}) -> - {ok, {Complete, N-1, Parts}}; -handle_create_msg({rexi_DOWN, _, _, _}, _, {Complete, _N, _Parts}) -> - if - Complete -> {stop, ok}; - true -> {error, create_db_fubar} - end; -handle_create_msg(_, _, {true, 1, _Acc}) -> - {stop, ok}; -handle_create_msg({ok, _}, {_, #shard{range=[Beg,_]}}, {false, 1, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - case is_complete(PartResults) of - true -> {stop, ok}; - false -> {error, create_db_fubar} - end; -handle_create_msg({ok, _}, _RefPart, {true, N, Parts}) -> - {ok, {true, N-1, Parts}}; -handle_create_msg({ok, _}, {_Ref, #shard{range=[Beg,_]}}, {false, Rem, PartResults0}) -> - PartResults = lists:keyreplace(Beg, 1, PartResults0, {Beg, true}), - {ok, {is_complete(PartResults), Rem-1, PartResults}}. +make_document([#shard{dbname=DbName}|_] = Shards) -> + {RawOut, ByNodeOut, ByRangeOut} = + lists:foldl(fun(#shard{node=N, range=[B,E]}, {Raw, ByNode, ByRange}) -> + Range = ?l2b([couch_util:to_hex(<>), "-", + couch_util:to_hex(<>)]), + Node = couch_util:to_binary(N), + {[[<<"add">>, Range, Node] | Raw], orddict:append(Node, Range, ByNode), + orddict:append(Range, Node, ByRange)} + end, {[], [], []}, Shards), + #doc{id=DbName, body = {[ + {<<"changelog">>, lists:sort(RawOut)}, + {<<"by_node">>, {[{K,lists:sort(V)} || {K,V} <- ByNodeOut]}}, + {<<"by_range">>, {[{K,lists:sort(V)} || {K,V} <- ByRangeOut]}} + ]}}. -is_complete(List) -> - lists:all(fun({_,Bool}) -> Bool end, List). +final_answer(Counters) -> + Successes = [X || {_, M} = X <- Counters, M == ok orelse M == file_exists], + case fabric_view:is_progress_possible(Successes) of + true -> + case lists:keymember(file_exists, 2, Successes) of + true -> + {error, file_exists}; + false -> + {stop, ok} + end; + false -> + {error, internal_server_error} + end. diff --git a/src/fabric_db_delete.erl b/src/fabric_db_delete.erl index 95b1a5ef..923b38dc 100644 --- a/src/fabric_db_delete.erl +++ b/src/fabric_db_delete.erl @@ -1,56 +1,40 @@ -module(fabric_db_delete). --author(brad@cloudant.com). - --export([delete_db/2]). +-export([go/2]). -include("fabric.hrl"). -%% @doc Delete a database, and all its partition files across the cluster -%% Options is proplist with user_ctx, n, q --spec delete_db(binary(), list()) -> {ok, #db{}} | {error, any()}. -delete_db(DbName, Options) -> - Fullmap = partitions:all_parts(DbName), - RefPartMap = send_delete_calls(Fullmap, Options), - Acc0 = {not_found, length(RefPartMap)}, - case fabric_util:receive_loop( - RefPartMap, 1, fun handle_delete_msg/3, Acc0) of - {ok, _Results} -> - delete_fullmap(DbName), +go(DbName, Options) -> + Shards = mem3:shards(DbName), + Workers = fabric_util:submit_jobs(Shards, delete_db, [Options, DbName]), + Acc0 = fabric_dict:init(Workers, nil), + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, ok} -> ok; - Error -> Error + {ok, not_found} -> + erlang:error(database_does_not_exist); + Error -> + Error end. -%% -%% internal -%% - -%% @doc delete the partitions on all appropriate nodes (rexi calls) --spec send_delete_calls(fullmap(), list()) -> [{reference(), part()}]. -send_delete_calls(Parts, Options) -> - lists:map(fun(#shard{node=Node, name=ShardName} = Part) -> - Ref = rexi:async_server_call({couch_server, Node}, - {delete, ShardName, Options}), - {Ref, Part} - end, Parts). - -handle_delete_msg(ok, _, {_, 1}) -> - {stop, ok}; -handle_delete_msg(not_found, _, {Acc, 1}) -> - {stop, Acc}; -handle_delete_msg({rexi_EXIT, _Reason}, _, {Acc, N}) -> - % TODO is this the appropriate action to take, or should we abort? - {ok, {Acc, N-1}}; -handle_delete_msg({rexi_DOWN, _, _, _}, _, _Acc) -> - {error, delete_db_fubar}; -handle_delete_msg(not_found, _, {Acc, N}) -> - {ok, {Acc, N-1}}; -handle_delete_msg(ok, _, {_, N}) -> - {ok, {ok, N-1}}. +handle_message(Msg, Shard, Counters) -> + C1 = fabric_dict:store(Shard, Msg, Counters), + case fabric_dict:any(nil, C1) of + true -> + {ok, C1}; + false -> + final_answer(C1) + end. -delete_fullmap(DbName) -> - case couch_db:open(<<"dbs">>, []) of - {ok, Db} -> - {ok, Doc} = couch_api:open_doc(Db, DbName, nil, []), - couch_api:update_doc(Db, Doc#doc{deleted=true}); - Error -> Error +final_answer(Counters) -> + Successes = [X || {_, M} = X <- Counters, M == ok orelse M == not_found], + case fabric_view:is_progress_possible(Successes) of + true -> + case lists:keymember(ok, 2, Successes) of + true -> + {stop, ok}; + false -> + {stop, not_found} + end; + false -> + {error, internal_server_error} end. diff --git a/src/fabric_db_doc_count.erl b/src/fabric_db_doc_count.erl index 4c3a72d5..c587d103 100644 --- a/src/fabric_db_doc_count.erl +++ b/src/fabric_db_doc_count.erl @@ -5,7 +5,7 @@ -include("fabric.hrl"). go(DbName) -> - Shards = partitions:all_parts(DbName), + Shards = mem3:shards(DbName), Workers = fabric_util:submit_jobs(Shards, get_doc_count, []), Acc0 = {length(Workers), [{Beg,nil} || #shard{range=[Beg,_]} <- Workers]}, fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). diff --git a/src/fabric_db_info.erl b/src/fabric_db_info.erl index 8db7212f..ecb8ce1c 100644 --- a/src/fabric_db_info.erl +++ b/src/fabric_db_info.erl @@ -5,7 +5,7 @@ -include("fabric.hrl"). go(DbName) -> - Shards = partitions:all_parts(DbName), + Shards = mem3:shards(DbName), Workers = fabric_util:submit_jobs(Shards, get_db_info, []), Acc0 = {fabric_dict:init(Workers, nil), []}, fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). diff --git a/src/fabric_doc_missing_revs.erl b/src/fabric_doc_missing_revs.erl index 8f1f5b4c..22c11ad6 100644 --- a/src/fabric_doc_missing_revs.erl +++ b/src/fabric_doc_missing_revs.erl @@ -51,7 +51,7 @@ group_idrevs_by_shard(DbName, IdsRevs) -> dict:to_list(lists:foldl(fun({Id, Revs}, D0) -> lists:foldl(fun(Shard, D1) -> dict:append(Shard, {Id, Revs}, D1) - end, D0, partitions:for_key(DbName,Id)) + end, D0, mem3:shards(DbName,Id)) end, dict:new(), IdsRevs)). update_dict(D0, KVs) -> diff --git a/src/fabric_doc_open.erl b/src/fabric_doc_open.erl index 6f39f39e..96545c55 100644 --- a/src/fabric_doc_open.erl +++ b/src/fabric_doc_open.erl @@ -5,7 +5,7 @@ -include("fabric.hrl"). go(DbName, Id, Options) -> - Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_doc, + Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_doc, [Id, Options]), SuppressDeletedDoc = not lists:member(deleted, Options), Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, diff --git a/src/fabric_doc_open_revs.erl b/src/fabric_doc_open_revs.erl index 2fa91208..5bf5499f 100644 --- a/src/fabric_doc_open_revs.erl +++ b/src/fabric_doc_open_revs.erl @@ -5,7 +5,7 @@ -include("fabric.hrl"). go(DbName, Id, Revs, Options) -> - Workers = fabric_util:submit_jobs(partitions:for_key(DbName,Id), open_revs, + Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_revs, [Id, Revs, Options]), Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index 9d3cd3d1..77c182f1 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -84,7 +84,7 @@ group_docs_by_shard(DbName, Docs) -> dict:to_list(lists:foldl(fun(#doc{id=Id} = Doc, D0) -> lists:foldl(fun(Shard, D1) -> dict:append(Shard, Doc, D1) - end, D0, partitions:for_key(DbName,Id)) + end, D0, mem3:shards(DbName,Id)) end, dict:new(), Docs)). append_update_replies([], [], DocReplyDict) -> diff --git a/src/fabric_group_info.erl b/src/fabric_group_info.erl index d2b76674..a1ba92cc 100644 --- a/src/fabric_group_info.erl +++ b/src/fabric_group_info.erl @@ -10,7 +10,7 @@ go(DbName, GroupId) when is_binary(GroupId) -> go(DbName, #doc{} = DDoc) -> Group = couch_view_group:design_doc_to_view_group(#db{name=DbName}, DDoc), - Shards = partitions:all_parts(DbName), + Shards = mem3:shards(DbName), Workers = fabric_util:submit_jobs(Shards, group_info, [Group]), Acc0 = {fabric_dict:init(Workers, nil), []}, fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 04aebfd1..1a2edf77 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -3,6 +3,7 @@ -export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). -export([all_docs/2, changes/3, map_view/4, reduce_view/4, group_info/2]). +-export([create_db/3, delete_db/3]). -include("fabric.hrl"). @@ -136,6 +137,19 @@ reduce_view(DbName, Group0, ViewName, QueryArgs) -> end, rexi:reply(complete). +create_db(DbName, Options, Doc) -> + mem3_util:write_db_doc(Doc), + rexi:reply(case couch_server:create(DbName, Options) of + {ok, _} -> + ok; + Error -> + Error + end). + +delete_db(DbName, Options, DocId) -> + mem3_util:delete_db_doc(DocId), + rexi:reply(couch_server:delete(DbName, Options)). + get_db_info(DbName) -> with_db(DbName, [], {couch_db, get_db_info, []}). diff --git a/src/fabric_view.erl b/src/fabric_view.erl index 738bb7dd..09fcd43c 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -8,34 +8,32 @@ %% @doc looks for a fully covered keyrange in the list of counters -spec is_progress_possible([{#shard{}, non_neg_integer()}]) -> boolean(). +is_progress_possible([]) -> + false; is_progress_possible(Counters) -> Ranges = fabric_dict:fold(fun(#shard{range=[X,Y]}, _, A) -> [{X,Y}|A] end, [], Counters), - [First | Rest] = lists:ukeysort(1, Ranges), - {Head, Tail} = lists:foldl(fun - (_, {Head, Tail}) when Head =:= Tail -> - % this is the success condition, we can fast-forward - {Head, Tail}; - (_, {foo, bar}) -> + [{0, Tail0} | Rest] = lists:ukeysort(1, Ranges), + Result = lists:foldl(fun + (_, fail) -> % we've already declared failure - {foo, bar}; - ({X,_}, {Head, Tail}) when Head < Tail, X > Tail -> + fail; + (_, complete) -> + % this is the success condition, we can fast-forward + complete; + ({X,_}, Tail) when X > (Tail+1) -> % gap in the keyrange, we're dead - {foo, bar}; - ({X,Y}, {Head, Tail}) when Head < Tail, X < Y -> - % the normal condition, adding to the tail - {Head, erlang:max(Tail, Y)}; - ({X,Y}, {Head, Tail}) when Head < Tail, X > Y, Y >= Head -> - % we've wrapped all the way around, trigger success condition - {Head, Head}; - ({X,Y}, {Head, Tail}) when Head < Tail, X > Y -> - % this wraps the keyspace, but there's still a gap. We're dead - % TODO technically, another shard could be a superset of this one, and - % we could still be alive. Pretty unlikely though, and impossible if - % we don't allow shards to wrap around the boundary - {foo, bar} - end, First, Rest), - Head =:= Tail. + fail; + ({_,Y}, Tail) -> + case erlang:max(Tail, Y) of + End when (End+1) =:= (2 bsl 31) -> + complete; + Else -> + % the normal condition, adding to the tail + Else + end + end, Tail0, Rest), + Result =:= complete. -spec remove_overlapping_shards(#shard{}, [#shard{}]) -> [#shard{}]. remove_overlapping_shards(#shard{range=[A,B]} = Shard0, Shards) -> diff --git a/src/fabric_view_all_docs.erl b/src/fabric_view_all_docs.erl index 196d6837..f1713b86 100644 --- a/src/fabric_view_all_docs.erl +++ b/src/fabric_view_all_docs.erl @@ -9,7 +9,7 @@ go(DbName, #view_query_args{keys=nil} = QueryArgs, Callback, Acc0) -> Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> Ref = rexi:cast(Node, {fabric_rpc, all_docs, [Name, QueryArgs]}), Shard#shard{ref = Ref} - end, partitions:all_parts(DbName)), + end, mem3:shards(DbName)), BufferSize = couch_config:get("fabric", "map_buffer_size", "2"), #view_query_args{limit = Limit, skip = Skip} = QueryArgs, State = #collector{ diff --git a/src/fabric_view_changes.erl b/src/fabric_view_changes.erl index 39a57176..f6989061 100644 --- a/src/fabric_view_changes.erl +++ b/src/fabric_view_changes.erl @@ -62,7 +62,7 @@ keep_sending_changes(DbName, Args, Callback, Seqs, AccIn, Timeout, TFun) -> end. send_changes(DbName, ChangesArgs, Callback, PackedSeqs, AccIn) -> - AllShards = partitions:all_parts(DbName), + AllShards = mem3:shards(DbName), Seqs = lists:flatmap(fun({#shard{name=Name, node=N} = Shard, Seq}) -> case lists:member(Shard, AllShards) of true -> @@ -168,7 +168,7 @@ make_changes_args(Options) -> get_start_seq(_DbName, #changes_args{dir=fwd, since=Since}) -> Since; get_start_seq(DbName, #changes_args{dir=rev}) -> - Shards = partitions:all_parts(DbName), + Shards = mem3:shards(DbName), Workers = fabric_util:submit_jobs(Shards, get_update_seq, []), {ok, Since} = fabric_util:recv(Workers, #shard.ref, fun collect_update_seqs/3, fabric_dict:init(Workers, -1)), @@ -195,10 +195,10 @@ pack_seqs(Workers) -> couch_util:encodeBase64Url(term_to_binary(SeqList, [compressed])). unpack_seqs(0, DbName) -> - fabric_dict:init(partitions:all_parts(DbName), 0); + fabric_dict:init(mem3:shards(DbName), 0); unpack_seqs("0", DbName) -> - fabric_dict:init(partitions:all_parts(DbName), 0); + fabric_dict:init(mem3:shards(DbName), 0); unpack_seqs(Packed, DbName) -> % TODO relies on internal structure of fabric_dict as keylist @@ -210,7 +210,7 @@ unpack_seqs(Packed, DbName) -> start_update_notifiers(DbName) -> lists:map(fun(#shard{node=Node, name=Name}) -> {Node, rexi:cast(Node, {?MODULE, start_update_notifier, [Name]})} - end, partitions:all_parts(DbName)). + end, mem3:shards(DbName)). % rexi endpoint start_update_notifier(DbName) -> diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl index 69133f3b..6c6dfc96 100644 --- a/src/fabric_view_map.erl +++ b/src/fabric_view_map.erl @@ -12,7 +12,7 @@ go(DbName, DDoc, View, Args, Callback, Acc0) -> Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> Ref = rexi:cast(Node, {fabric_rpc, map_view, [Name, DDoc, View, Args]}), Shard#shard{ref = Ref} - end, partitions:all_parts(DbName)), + end, mem3:shards(DbName)), BufferSize = couch_config:get("fabric", "map_buffer_size", "2"), #view_query_args{limit = Limit, skip = Skip, keys = Keys} = Args, State = #collector{ diff --git a/src/fabric_view_reduce.erl b/src/fabric_view_reduce.erl index ca137314..4f08b3ed 100644 --- a/src/fabric_view_reduce.erl +++ b/src/fabric_view_reduce.erl @@ -16,7 +16,7 @@ go(DbName, DDoc, VName, Args, Callback, Acc0) -> Workers = lists:map(fun(#shard{name=Name, node=N} = Shard) -> Ref = rexi:cast(N, {fabric_rpc, reduce_view, [Name,Group,VName,Args]}), Shard#shard{ref = Ref} - end, partitions:all_parts(DbName)), + end, mem3:shards(DbName)), BufferSize = couch_config:get("fabric", "reduce_buffer_size", "20"), #view_query_args{limit = Limit, skip = Skip} = Args, State = #collector{ -- cgit v1.2.3 From bbf3c93a949b18c31d5a16057c122f3cffa94c35 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 2 Jul 2010 03:12:23 -0400 Subject: use name_shard/1 instead of obsolete shard_name/2 --- src/fabric_view_changes.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabric_view_changes.erl b/src/fabric_view_changes.erl index f6989061..3e8339bd 100644 --- a/src/fabric_view_changes.erl +++ b/src/fabric_view_changes.erl @@ -203,8 +203,8 @@ unpack_seqs("0", DbName) -> unpack_seqs(Packed, DbName) -> % TODO relies on internal structure of fabric_dict as keylist lists:map(fun({Node, [A,B], Seq}) -> - Name = partitions:shard_name(A, DbName), - {#shard{node=Node, range=[A,B], dbname=DbName, name=Name}, Seq} + Shard = #shard{node=Node, range=[A,B], dbname=DbName}, + {mem3_util:name_shard(Shard), Seq} end, binary_to_term(couch_util:decodeBase64Url(Packed))). start_update_notifiers(DbName) -> -- cgit v1.2.3 From 2daeaacda8cb4090df3d981ba4e5198851b76dda Mon Sep 17 00:00:00 2001 From: Brad Anderson Date: Fri, 2 Jul 2010 14:20:37 -0400 Subject: no authors in src files --- src/fabric_all_databases.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fabric_all_databases.erl b/src/fabric_all_databases.erl index 4bb5c5a5..b93f483c 100644 --- a/src/fabric_all_databases.erl +++ b/src/fabric_all_databases.erl @@ -1,5 +1,4 @@ -module(fabric_all_databases). --author(brad@cloudant.com). -export([all_databases/1]). -- cgit v1.2.3 From 07cc904637aedf1ba0577a2aaa4574f593ef0007 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 2 Jul 2010 12:53:53 -0400 Subject: better error logging for failed RPC requests --- src/fabric_rpc.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 1a2edf77..948bc54b 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -203,7 +203,15 @@ group_info(DbName, Group0) -> with_db(DbName, Options, {M,F,A}) -> case couch_db:open(DbName, Options) of {ok, Db} -> - rexi:reply(apply(M, F, [Db | A])); + rexi:reply(try + apply(M, F, [Db | A]) + catch Exception -> + Exception; + error:Reason -> + ?LOG_ERROR("~p ~p ~p~n~p", [?MODULE, {M,F}, Reason, + erlang:get_stacktrace()]), + {error, Reason} + end); Error -> rexi:reply(Error) end. -- cgit v1.2.3 From 6e6d8021855cb797e0312164d27b9e00b3c29411 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 2 Jul 2010 14:21:40 -0400 Subject: convenience function to reset DB validation functions --- src/fabric.erl | 6 +++++- src/fabric_rpc.erl | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index b233677b..9545e729 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -13,7 +13,7 @@ get_view_group_info/2]). % miscellany --export([db_path/2, design_docs/1]). +-export([db_path/2, design_docs/1, reset_validation_funs/1]). -include("fabric.hrl"). @@ -120,6 +120,10 @@ design_docs(DbName) -> end, fabric:all_docs(dbname(DbName), QueryArgs, Callback, []). +reset_validation_funs(DbName) -> + [rexi:cast(Node, {fabric_rpc, reset_validation_funs, [Name]}) || + #shard{node=Node, name=Name} <- mem3:shards(DbName)]. + %% some simple type validation and transcoding dbname(DbName) when is_list(DbName) -> diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 948bc54b..2baaf49a 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -3,7 +3,7 @@ -export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). -export([all_docs/2, changes/3, map_view/4, reduce_view/4, group_info/2]). --export([create_db/3, delete_db/3]). +-export([create_db/3, delete_db/3, reset_validation_funs/1]). -include("fabric.hrl"). @@ -196,6 +196,14 @@ group_info(DbName, Group0) -> {ok, Pid} = gen_server:call(couch_view, {get_group_server, DbName, Group0}), rexi:reply(couch_view_group:request_group_info(Pid)). +reset_validation_funs(DbName) -> + case couch_db:open(DbName, []) of + {ok, Db} -> + gen_server:cast(Db#db.update_pid, {load_validation_funs, undefined}); + _ -> + ok + end. + %% %% internal %% -- cgit v1.2.3 From 34d0454da8df940884804a82405ae03638816abc Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Sun, 4 Jul 2010 10:51:38 -0400 Subject: talk to main_pid, not update_pid --- src/fabric_rpc.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 2baaf49a..e2dc88b9 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -198,8 +198,8 @@ group_info(DbName, Group0) -> reset_validation_funs(DbName) -> case couch_db:open(DbName, []) of - {ok, Db} -> - gen_server:cast(Db#db.update_pid, {load_validation_funs, undefined}); + {ok, #db{main_pid = Pid}} -> + gen_server:cast(Pid, {load_validation_funs, undefined}); _ -> ok end. -- cgit v1.2.3 From a21c9579dccf64981258a62e892d69804b05dcbf Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 5 Jul 2010 11:00:24 -0400 Subject: really implement write quorum checking --- src/fabric_doc_update.erl | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index 77c182f1..ea7d8d74 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -10,8 +10,10 @@ go(DbName, AllDocs, Options) -> {Shard#shard{ref=Ref}, Docs} end, group_docs_by_shard(DbName, AllDocs)), {Workers, _} = lists:unzip(GroupedDocs), - Acc0 = {length(Workers), length(AllDocs), couch_util:get_value(w, Options, 1), - GroupedDocs, dict:from_list([{Doc,[]} || Doc <- AllDocs])}, + W = list_to_integer(couch_util:get_value(w, Options, + couch_config:get("cluster", "w", "2"))), + Acc0 = {length(Workers), length(AllDocs), W, GroupedDocs, + dict:from_list([{Doc,[]} || Doc <- AllDocs])}, case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of {ok, Results} -> Reordered = couch_util:reorder_results(AllDocs, Results), @@ -50,13 +52,16 @@ handle_message({not_found, no_db_file} = X, Worker, Acc0) -> Docs = couch_util:get_value(Worker, GroupedDocs), handle_message({ok, [X || _D <- Docs]}, Worker, Acc0). -force_reply(Doc, Replies, {W, Acc}) -> - % TODO make a real decision here - case Replies of - [] -> - {W, [{Doc, {error, internal_server_error}} | Acc]}; - [Reply| _] -> - {W, [{Doc, Reply} | Acc]} +force_reply(Doc, [], {W, Acc}) -> + {W, [{Doc, {error, internal_server_error}} | Acc]}; +force_reply(Doc, [FirstReply|_] = Replies, {W, Acc}) -> + case update_quorum_met(W, Replies) of + {true, Reply} -> + {W, [{Doc,Reply} | Acc]}; + false -> + ?LOG_ERROR("write quorum (~p) failed, reply ~p", [W, FirstReply]), + % TODO make a smarter choice than just picking the first reply + {W, [{Doc,FirstReply} | Acc]} end. maybe_reply(_, _, continue) -> @@ -71,12 +76,13 @@ maybe_reply(Doc, Replies, {stop, W, Acc}) -> end. update_quorum_met(W, Replies) -> - % TODO make a real decision here - case length(Replies) >= W of - true -> - {true, hd(Replies)}; - false -> - false + Counters = lists:foldl(fun(R,D) -> orddict:update_counter(R,1,D) end, + orddict:new(), Replies), + case lists:dropwhile(fun({_, Count}) -> Count < W end, Counters) of + [] -> + false; + [{FinalReply, _} | _] -> + {true, FinalReply} end. -spec group_docs_by_shard(binary(), [#doc{}]) -> [{#shard{}, [#doc{}]}]. -- cgit v1.2.3 From fecc6c897c5d8385e7786662f62a642a9d84db69 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 5 Jul 2010 11:17:54 -0400 Subject: respect default r quorum in cluster config --- src/fabric_doc_open.erl | 3 ++- src/fabric_doc_open_revs.erl | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fabric_doc_open.erl b/src/fabric_doc_open.erl index 96545c55..19509295 100644 --- a/src/fabric_doc_open.erl +++ b/src/fabric_doc_open.erl @@ -8,7 +8,8 @@ go(DbName, Id, Options) -> Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_doc, [Id, Options]), SuppressDeletedDoc = not lists:member(deleted, Options), - Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, + R = couch_util:get_value(r, Options, couch_config:get("cluster","r","2")), + Acc0 = {length(Workers), list_to_integer(R), []}, case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of {ok, {ok, #doc{deleted=true}}} when SuppressDeletedDoc -> {not_found, deleted}; diff --git a/src/fabric_doc_open_revs.erl b/src/fabric_doc_open_revs.erl index 5bf5499f..9d3dae41 100644 --- a/src/fabric_doc_open_revs.erl +++ b/src/fabric_doc_open_revs.erl @@ -7,7 +7,8 @@ go(DbName, Id, Revs, Options) -> Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_revs, [Id, Revs, Options]), - Acc0 = {length(Workers), couch_util:get_value(r, Options, 1), []}, + R = couch_util:get_value(r, Options, couch_config:get("cluster","r","2")), + Acc0 = {length(Workers), list_to_integer(R), []}, case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of {ok, {ok, Reply}} -> {ok, Reply}; -- cgit v1.2.3 From 45c3f81d448167e00530e103488e115c52c54c6b Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 5 Jul 2010 11:19:01 -0400 Subject: purely cosmetic --- src/fabric_doc_update.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index ea7d8d74..05b6a785 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -10,9 +10,8 @@ go(DbName, AllDocs, Options) -> {Shard#shard{ref=Ref}, Docs} end, group_docs_by_shard(DbName, AllDocs)), {Workers, _} = lists:unzip(GroupedDocs), - W = list_to_integer(couch_util:get_value(w, Options, - couch_config:get("cluster", "w", "2"))), - Acc0 = {length(Workers), length(AllDocs), W, GroupedDocs, + W = couch_util:get_value(w, Options, couch_config:get("cluster","w","2")), + Acc0 = {length(Workers), length(AllDocs), list_to_integer(W), GroupedDocs, dict:from_list([{Doc,[]} || Doc <- AllDocs])}, case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of {ok, Results} -> -- cgit v1.2.3 From 22a487d1939dad5ba5032494427950afb2493795 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 5 Jul 2010 15:34:02 -0400 Subject: update include path for couch_db.hrl --- include/fabric.hrl | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/include/fabric.hrl b/include/fabric.hrl index 5cb337d9..4347ef82 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -1,14 +1,5 @@ --define(FABRIC, true). - --ifndef(COUCH). --include("../../couch/src/couch_db.hrl"). --endif. - --ifndef(MEMBERSHIP). --include("../../mem3/include/mem3.hrl"). --endif. - -include_lib("eunit/include/eunit.hrl"). +-include_lib("mem3/include/mem3.hrl"). -record(collector, { query_args, -- cgit v1.2.3 From 107e7b24b83bdeee98a3b078150921d12875f56b Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 5 Jul 2010 15:49:43 -0400 Subject: more cleanup of the includes --- include/fabric.hrl | 1 - src/fabric.erl | 3 +++ src/fabric_all_databases.erl | 3 ++- src/fabric_db_create.erl | 2 ++ src/fabric_db_delete.erl | 1 + src/fabric_db_doc_count.erl | 2 ++ src/fabric_db_info.erl | 1 + src/fabric_doc_missing_revs.erl | 1 + src/fabric_doc_open.erl | 2 ++ src/fabric_doc_open_revs.erl | 2 ++ src/fabric_doc_update.erl | 2 ++ src/fabric_group_info.erl | 2 ++ src/fabric_rpc.erl | 1 + src/fabric_util.erl | 1 + src/fabric_view.erl | 2 ++ src/fabric_view_all_docs.erl | 2 ++ src/fabric_view_changes.erl | 2 ++ src/fabric_view_map.erl | 2 ++ src/fabric_view_reduce.erl | 2 ++ 19 files changed, 32 insertions(+), 2 deletions(-) diff --git a/include/fabric.hrl b/include/fabric.hrl index 4347ef82..6ec17b34 100644 --- a/include/fabric.hrl +++ b/include/fabric.hrl @@ -1,5 +1,4 @@ -include_lib("eunit/include/eunit.hrl"). --include_lib("mem3/include/mem3.hrl"). -record(collector, { query_args, diff --git a/src/fabric.erl b/src/fabric.erl index 9545e729..9be9a26b 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -1,5 +1,8 @@ -module(fabric). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). + % DBs -export([all_dbs/0, all_dbs/1, create_db/1, create_db/2, delete_db/1, delete_db/2, get_db_info/1, get_doc_count/1]). diff --git a/src/fabric_all_databases.erl b/src/fabric_all_databases.erl index b93f483c..97685f4e 100644 --- a/src/fabric_all_databases.erl +++ b/src/fabric_all_databases.erl @@ -3,6 +3,7 @@ -export([all_databases/1]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). %% @doc gets all databases in the cluster. -spec all_databases(binary() | []) -> [binary()]. @@ -14,7 +15,7 @@ all_databases([]) -> all_databases(Customer) -> ?debugFmt("~nCustomer: ~p~n", [Customer]), Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> - DbNameStr = ?b2l(DbName), + DbNameStr = binary_to_list(DbName), case string:str(DbNameStr, Customer) of 1 -> new_acc(DbNameStr, AccIn); diff --git a/src/fabric_db_create.erl b/src/fabric_db_create.erl index 80bd1eb1..2fca0eed 100644 --- a/src/fabric_db_create.erl +++ b/src/fabric_db_create.erl @@ -2,6 +2,8 @@ -export([go/2]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). %% @doc Create a new database, and all its partition files across the cluster %% Options is proplist with user_ctx, n, q diff --git a/src/fabric_db_delete.erl b/src/fabric_db_delete.erl index 923b38dc..57eefa9e 100644 --- a/src/fabric_db_delete.erl +++ b/src/fabric_db_delete.erl @@ -2,6 +2,7 @@ -export([go/2]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). go(DbName, Options) -> Shards = mem3:shards(DbName), diff --git a/src/fabric_db_doc_count.erl b/src/fabric_db_doc_count.erl index c587d103..332b923d 100644 --- a/src/fabric_db_doc_count.erl +++ b/src/fabric_db_doc_count.erl @@ -3,6 +3,8 @@ -export([go/1]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName) -> Shards = mem3:shards(DbName), diff --git a/src/fabric_db_info.erl b/src/fabric_db_info.erl index ecb8ce1c..3758c5c3 100644 --- a/src/fabric_db_info.erl +++ b/src/fabric_db_info.erl @@ -3,6 +3,7 @@ -export([go/1]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). go(DbName) -> Shards = mem3:shards(DbName), diff --git a/src/fabric_doc_missing_revs.erl b/src/fabric_doc_missing_revs.erl index 22c11ad6..9a368783 100644 --- a/src/fabric_doc_missing_revs.erl +++ b/src/fabric_doc_missing_revs.erl @@ -3,6 +3,7 @@ -export([go/2]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). go(DbName, AllIdsRevs) -> Workers = lists:map(fun({#shard{name=Name, node=Node} = Shard, IdsRevs}) -> diff --git a/src/fabric_doc_open.erl b/src/fabric_doc_open.erl index 19509295..16493fb6 100644 --- a/src/fabric_doc_open.erl +++ b/src/fabric_doc_open.erl @@ -3,6 +3,8 @@ -export([go/3]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName, Id, Options) -> Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_doc, diff --git a/src/fabric_doc_open_revs.erl b/src/fabric_doc_open_revs.erl index 9d3dae41..61ff466f 100644 --- a/src/fabric_doc_open_revs.erl +++ b/src/fabric_doc_open_revs.erl @@ -3,6 +3,8 @@ -export([go/4]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName, Id, Revs, Options) -> Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_revs, diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index 05b6a785..f7a91d48 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -3,6 +3,8 @@ -export([go/3]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName, AllDocs, Options) -> GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> diff --git a/src/fabric_group_info.erl b/src/fabric_group_info.erl index a1ba92cc..04605a66 100644 --- a/src/fabric_group_info.erl +++ b/src/fabric_group_info.erl @@ -3,6 +3,8 @@ -export([go/2]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName, GroupId) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, GroupId, []), diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index e2dc88b9..673627da 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -6,6 +6,7 @@ -export([create_db/3, delete_db/3, reset_validation_funs/1]). -include("fabric.hrl"). +-include_lib("couch/include/couch_db.hrl"). -record (view_acc, { db, diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 90b0c647..3c27f19e 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -3,6 +3,7 @@ -export([submit_jobs/3, cleanup/1, recv/4, receive_loop/4, receive_loop/6]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). submit_jobs(Shards, EndPoint, ExtraArgs) -> lists:map(fun(#shard{node=Node, name=ShardName} = Shard) -> diff --git a/src/fabric_view.erl b/src/fabric_view.erl index 09fcd43c..6817a5b0 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -5,6 +5,8 @@ extract_view/4]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). %% @doc looks for a fully covered keyrange in the list of counters -spec is_progress_possible([{#shard{}, non_neg_integer()}]) -> boolean(). diff --git a/src/fabric_view_all_docs.erl b/src/fabric_view_all_docs.erl index f1713b86..d51a2831 100644 --- a/src/fabric_view_all_docs.erl +++ b/src/fabric_view_all_docs.erl @@ -4,6 +4,8 @@ -export([open_doc/3]). % exported for spawn -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName, #view_query_args{keys=nil} = QueryArgs, Callback, Acc0) -> Workers = lists:map(fun(#shard{name=Name, node=Node} = Shard) -> diff --git a/src/fabric_view_changes.erl b/src/fabric_view_changes.erl index 3e8339bd..39755a01 100644 --- a/src/fabric_view_changes.erl +++ b/src/fabric_view_changes.erl @@ -3,6 +3,8 @@ -export([go/5, start_update_notifier/1]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName, Feed, Options, Callback, Acc0) when Feed == "continuous" orelse Feed == "longpoll" -> diff --git a/src/fabric_view_map.erl b/src/fabric_view_map.erl index 6c6dfc96..ce8dd625 100644 --- a/src/fabric_view_map.erl +++ b/src/fabric_view_map.erl @@ -3,6 +3,8 @@ -export([go/6]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName, GroupId, View, Args, Callback, Acc0) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), diff --git a/src/fabric_view_reduce.erl b/src/fabric_view_reduce.erl index 4f08b3ed..ddde9f22 100644 --- a/src/fabric_view_reduce.erl +++ b/src/fabric_view_reduce.erl @@ -3,6 +3,8 @@ -export([go/6]). -include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). +-include_lib("couch/include/couch_db.hrl"). go(DbName, GroupId, View, Args, Callback, Acc0) when is_binary(GroupId) -> {ok, DDoc} = fabric:open_doc(DbName, <<"_design/", GroupId/binary>>, []), -- cgit v1.2.3 From 8ff2af56e04258a5a8fb3a6c35bb824f0e8b9bc6 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 5 Jul 2010 17:00:55 -0400 Subject: move db_path logic to chttpd --- src/fabric.erl | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index 9be9a26b..21879f43 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -16,16 +16,11 @@ get_view_group_info/2]). % miscellany --export([db_path/2, design_docs/1, reset_validation_funs/1]). +-export([design_docs/1, reset_validation_funs/1]). -include("fabric.hrl"). % db operations --spec db_path(bstring(), bstring()) -> bstring(). -db_path(RawUri, Customer) -> - CustomerUri = generate_customer_path(RawUri, Customer), - {Path, _, _} = mochiweb_util:urlsplit_path(CustomerUri), - Path. all_dbs() -> fabric_all_databases:all_databases(""). @@ -197,17 +192,3 @@ default_callback(Row, Acc) -> is_reduce_view(_, _, _, #view_query_args{view_type=Reduce}) -> Reduce =:= reduce. - -generate_customer_path("/", _Customer) -> - ""; -generate_customer_path("/favicon.ico", _Customer) -> - "favicon.ico"; -generate_customer_path([$/,$_|Rest], _Customer) -> - lists:flatten([$_|Rest]); -generate_customer_path([$/|RawPath], Customer) -> - case Customer of - "" -> - RawPath; - Else -> - lists:flatten([Else, "%2F", RawPath]) - end. -- cgit v1.2.3 From 733c58a1e9f776f5f06c0c340571b6025276ffad Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 6 Jul 2010 11:40:38 -0400 Subject: rearrange fabric arguments to match couch_btree --- src/fabric.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index 21879f43..892673a5 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -73,11 +73,11 @@ update_docs(DbName, Docs, Options) -> att_receiver(Req, Length) -> fabric_doc_attachments:receiver(Req, Length). -all_docs(DbName, #view_query_args{} = QueryArgs, Callback, Acc0) when +all_docs(DbName, Callback, Acc0, #view_query_args{} = QueryArgs) when is_function(Callback, 2) -> fabric_view_all_docs:go(dbname(DbName), QueryArgs, Callback, Acc0). -changes(DbName, Options, Callback, Acc0) -> +changes(DbName, Callback, Acc0, Options) -> % TODO use a keylist for Options instead of #changes_args, BugzID 10281 Feed = Options#changes_args.feed, fabric_view_changes:go(dbname(DbName), Feed, Options, Callback, Acc0). @@ -87,9 +87,9 @@ query_view(DbName, DesignName, ViewName) -> query_view(DbName, DesignName, ViewName, QueryArgs) -> Callback = fun default_callback/2, - query_view(DbName, DesignName, ViewName, QueryArgs, Callback, []). + query_view(DbName, DesignName, ViewName, Callback, [], QueryArgs). -query_view(DbName, Design, ViewName, QueryArgs, Callback, Acc0) -> +query_view(DbName, Design, ViewName, Callback, Acc0, QueryArgs) -> Db = dbname(DbName), View = name(ViewName), case is_reduce_view(Db, Design, View, QueryArgs) of true -> @@ -116,7 +116,7 @@ design_docs(DbName) -> (complete, Acc) -> {ok, lists:reverse(Acc)} end, - fabric:all_docs(dbname(DbName), QueryArgs, Callback, []). + fabric:all_docs(dbname(DbName), Callback, [], QueryArgs). reset_validation_funs(DbName) -> [rexi:cast(Node, {fabric_rpc, reset_validation_funs, [Name]}) || -- cgit v1.2.3 From dc4f15e586112742cf54628e937e0f776022c953 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Sat, 10 Jul 2010 15:50:17 -0400 Subject: thank you dialyzer --- src/fabric_all_databases.erl | 2 +- src/fabric_util.erl | 2 +- src/fabric_view.erl | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/fabric_all_databases.erl b/src/fabric_all_databases.erl index 97685f4e..e16429b1 100644 --- a/src/fabric_all_databases.erl +++ b/src/fabric_all_databases.erl @@ -6,7 +6,7 @@ -include_lib("mem3/include/mem3.hrl"). %% @doc gets all databases in the cluster. --spec all_databases(binary() | []) -> [binary()]. +-spec all_databases(string()) -> {ok, [binary()]}. all_databases([]) -> Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> new_acc(DbName, AccIn) diff --git a/src/fabric_util.erl b/src/fabric_util.erl index 3c27f19e..e928c5ff 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -28,7 +28,7 @@ receive_loop(Workers, Keypos, Fun, Acc0) -> %% @doc set up the receive loop with an overall timeout -spec receive_loop([any()], integer(), function(), any(), timeout(), timeout()) -> - {ok, any()}. + {ok, any()} | timeout | {error, any()}. receive_loop(RefPartMap, Keypos, Fun, Acc0, infinity, PerMsgTO) -> process_mailbox(RefPartMap, Keypos, Fun, Acc0, nil, PerMsgTO); receive_loop(RefPartMap, Keypos, Fun, Acc0, GlobalTimeout, PerMsgTO) -> diff --git a/src/fabric_view.erl b/src/fabric_view.erl index 6817a5b0..4f29abf7 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -9,7 +9,7 @@ -include_lib("couch/include/couch_db.hrl"). %% @doc looks for a fully covered keyrange in the list of counters --spec is_progress_possible([{#shard{}, non_neg_integer()}]) -> boolean(). +-spec is_progress_possible([{#shard{}, term()}]) -> boolean(). is_progress_possible([]) -> false; is_progress_possible(Counters) -> @@ -37,7 +37,8 @@ is_progress_possible(Counters) -> end, Tail0, Rest), Result =:= complete. --spec remove_overlapping_shards(#shard{}, [#shard{}]) -> [#shard{}]. +-spec remove_overlapping_shards(#shard{}, [{#shard{}, any()}]) -> + [{#shard{}, any()}]. remove_overlapping_shards(#shard{range=[A,B]} = Shard0, Shards) -> fabric_dict:filter(fun(#shard{range=[X,Y]} = Shard, _Value) -> if Shard =:= Shard0 -> -- cgit v1.2.3 From ab01f6fc3c50279953838d675067083a8cf0799f Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 12 Jul 2010 12:36:54 -0400 Subject: fix fabric:all_dbs when used with a prefix --- ebin/fabric.app | 1 - src/fabric.erl | 19 +++++++++++++++---- src/fabric_all_databases.erl | 36 ------------------------------------ 3 files changed, 15 insertions(+), 41 deletions(-) delete mode 100644 src/fabric_all_databases.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index 6750766b..716591d1 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -5,7 +5,6 @@ {vsn, "1.0"}, {modules, [ fabric, - fabric_all_databases, fabric_db_create, fabric_db_delete, fabric_db_doc_count, diff --git a/src/fabric.erl b/src/fabric.erl index 892673a5..9158b9e9 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -23,10 +23,21 @@ % db operations all_dbs() -> - fabric_all_databases:all_databases(""). - -all_dbs(Customer) -> - fabric_all_databases:all_databases(Customer). + all_dbs(<<>>). + +all_dbs(Prefix) when is_list(Prefix) -> + all_dbs(list_to_binary(Prefix)); +all_dbs(Prefix) when is_binary(Prefix) -> + Length = byte_size(Prefix), + MatchingDbs = ets:foldl(fun(#shard{dbname=DbName}, Acc) -> + case DbName of + <> -> + [DbName | Acc]; + _ -> + Acc + end + end, [], partitions), + {ok, lists:usort(MatchingDbs)}. get_db_info(DbName) -> fabric_db_info:go(dbname(DbName)). diff --git a/src/fabric_all_databases.erl b/src/fabric_all_databases.erl deleted file mode 100644 index e16429b1..00000000 --- a/src/fabric_all_databases.erl +++ /dev/null @@ -1,36 +0,0 @@ --module(fabric_all_databases). - --export([all_databases/1]). - --include("fabric.hrl"). --include_lib("mem3/include/mem3.hrl"). - -%% @doc gets all databases in the cluster. --spec all_databases(string()) -> {ok, [binary()]}. -all_databases([]) -> - Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> - new_acc(DbName, AccIn) - end, [], partitions), - {ok, Dbs}; -all_databases(Customer) -> - ?debugFmt("~nCustomer: ~p~n", [Customer]), - Dbs = ets:foldl(fun(#shard{dbname=DbName}, AccIn) -> - DbNameStr = binary_to_list(DbName), - case string:str(DbNameStr, Customer) of - 1 -> - new_acc(DbNameStr, AccIn); - _ -> AccIn - end - end, [], dbs_cache), - {ok, Dbs}. - - -%% ===================== -%% internal -%% ===================== - -new_acc(DbName, Acc) -> - case lists:member(DbName, Acc) of - true -> Acc; - _ ->[DbName | Acc] - end. -- cgit v1.2.3 From 3dd9e521192458002efc7ddaaa7455bcc549afd6 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 12 Jul 2010 17:20:35 -0400 Subject: handle new_edits=false update_doc response correctly --- src/fabric.erl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index 9158b9e9..9b74b691 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -70,12 +70,15 @@ get_missing_revs(DbName, IdsRevs) when is_list(IdsRevs) -> fabric_doc_missing_revs:go(dbname(DbName), Sanitized). update_doc(DbName, Doc, Options) -> - {ok, [Result]} = update_docs(DbName, [Doc], opts(Options)), - case Result of - {ok, _} -> - Result; - Error -> - throw(Error) + case update_docs(DbName, [Doc], opts(Options)) of + {ok, [{ok, NewRev}]} -> + {ok, NewRev}; + {ok, [Error]} -> + throw(Error); + {ok, []} -> + % replication success + #doc{revs = {Pos, [RevId | _]}} = doc(Doc), + {ok, {Pos, RevId}} end. update_docs(DbName, Docs, Options) -> -- cgit v1.2.3 From 0df44a1122d7fd99f96992946692737361941b64 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 13 Jul 2010 19:48:07 -0400 Subject: return an informative error if user tries unsupported all_or_nothing option --- src/fabric.erl | 5 ++++- src/fabric_doc_update.erl | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/fabric.erl b/src/fabric.erl index 9b74b691..5d03c5fa 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -82,7 +82,10 @@ update_doc(DbName, Doc, Options) -> end. update_docs(DbName, Docs, Options) -> - fabric_doc_update:go(dbname(DbName), docs(Docs), opts(Options)). + try fabric_doc_update:go(dbname(DbName), docs(Docs), opts(Options)) + catch {aborted, PreCommitFailures} -> + {aborted, PreCommitFailures} + end. att_receiver(Req, Length) -> fabric_doc_attachments:receiver(Req, Length). diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index f7a91d48..0898d96c 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -6,7 +6,9 @@ -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). -go(DbName, AllDocs, Options) -> +go(DbName, AllDocs, Opts) -> + validate_atomic_update(DbName, AllDocs, lists:member(all_or_nothing, Opts)), + Options = lists:delete(all_or_nothing, Opts), GroupedDocs = lists:map(fun({#shard{name=Name, node=Node} = Shard, Docs}) -> Ref = rexi:cast(Node, {fabric_rpc, update_docs, [Name, Docs, Options]}), {Shard#shard{ref=Ref}, Docs} @@ -107,3 +109,15 @@ skip_message(Acc0) -> % TODO fix this {ok, Acc0}. +validate_atomic_update(_, _, false) -> + ok; +validate_atomic_update(_DbName, AllDocs, true) -> + % TODO actually perform the validation. This requires some hackery, we need + % to basically extract the prep_and_validate_updates function from couch_db + % and only run that, without actually writing in case of a success. + Error = {not_implemented, <<"all_or_nothing is not supported yet">>}, + PreCommitFailures = lists:map(fun(#doc{id=Id, revs = {Pos,Revs}}) -> + case Revs of [] -> RevId = <<>>; [RevId|_] -> ok end, + {{Id, {Pos, RevId}}, Error} + end, AllDocs), + throw({aborted, PreCommitFailures}). -- cgit v1.2.3 From 24850155eee3a969d8f2daf127a5a782f6c273cd Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 14 Jul 2010 09:55:00 -0400 Subject: fabric:get_doc_count/1 was broken for N>1 --- src/fabric_db_doc_count.erl | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/src/fabric_db_doc_count.erl b/src/fabric_db_doc_count.erl index 332b923d..12d5cbf8 100644 --- a/src/fabric_db_doc_count.erl +++ b/src/fabric_db_doc_count.erl @@ -9,36 +9,24 @@ go(DbName) -> Shards = mem3:shards(DbName), Workers = fabric_util:submit_jobs(Shards, get_doc_count, []), - Acc0 = {length(Workers), [{Beg,nil} || #shard{range=[Beg,_]} <- Workers]}, + Acc0 = {fabric_dict:init(Workers, nil), 0}, fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0). -handle_message({ok, Count}, #shard{range=[Beg,_]}, {N, Infos0}) -> - case couch_util:get_value(Beg, Infos0) of +handle_message({ok, Count}, Shard, {Counters, Acc}) -> + case fabric_dict:lookup_element(Shard, Counters) of + undefined -> + % already heard from someone else in this range + {ok, {Counters, Acc}}; nil -> - Infos = lists:keyreplace(Beg, 1, Infos0, {Beg, Count}), - case is_complete(Infos) of + C1 = fabric_dict:store(Shard, ok, Counters), + C2 = fabric_view:remove_overlapping_shards(Shard, C1), + case fabric_dict:any(nil, C2) of true -> - {stop, lists:sum([C || {_, C} <- Infos])}; + {ok, {C2, Count+Acc}}; false -> - if N > 1 -> - {ok, {N-1, Infos}}; - true -> - report_error(Infos), - {error, missing_shards} - end - end; - _ -> - {ok, {N-1, Infos0}} + {stop, Count+Acc} + end end; -handle_message(_, _, {1, Infos}) -> - report_error(Infos), - {error, missing_shards}; -handle_message(_Other, _, {N, Infos0}) -> - {ok, {N-1, Infos0}}. +handle_message(_, _, Acc) -> + {ok, Acc}. -report_error(Infos) -> - MissingShards = [S || {S,nil} <- Infos], - ?LOG_ERROR("doc_count error, missing shards: ~p", [MissingShards]). - -is_complete(List) -> - not lists:keymember(nil, 2, List). -- cgit v1.2.3 From 63330eb0bd922f504fb7b4111f98236879711288 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 14 Jul 2010 13:32:19 -0400 Subject: support the atts_since qs param for GET requests --- src/fabric_doc_open.erl | 2 +- src/fabric_rpc.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabric_doc_open.erl b/src/fabric_doc_open.erl index 16493fb6..5c5699c3 100644 --- a/src/fabric_doc_open.erl +++ b/src/fabric_doc_open.erl @@ -8,7 +8,7 @@ go(DbName, Id, Options) -> Workers = fabric_util:submit_jobs(mem3:shards(DbName,Id), open_doc, - [Id, Options]), + [Id, [deleted|Options]]), SuppressDeletedDoc = not lists:member(deleted, Options), R = couch_util:get_value(r, Options, couch_config:get("cluster","r","2")), Acc0 = {length(Workers), list_to_integer(R), []}, diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 673627da..6be7c3cc 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -161,7 +161,7 @@ get_update_seq(DbName) -> with_db(DbName, [], {couch_db, get_update_seq, []}). open_doc(DbName, DocId, Options) -> - with_db(DbName, Options, {couch_db, open_doc_int, [DocId, Options]}). + with_db(DbName, Options, {couch_db, open_doc, [DocId, Options]}). open_revs(DbName, Id, Revs, Options) -> with_db(DbName, Options, {couch_db, open_doc_revs, [Id, Revs, Options]}). -- cgit v1.2.3 From 5c1c2422dcf9673d67681f999ece72a019461306 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 14 Jul 2010 14:49:22 -0400 Subject: support for revs_limit and security metadata --- ebin/fabric.app | 1 + src/fabric.erl | 16 +++++++++++++++- src/fabric_db_meta.erl | 30 ++++++++++++++++++++++++++++++ src/fabric_rpc.erl | 9 ++++++++- src/fabric_util.erl | 14 +++++++++++++- 5 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/fabric_db_meta.erl diff --git a/ebin/fabric.app b/ebin/fabric.app index 716591d1..bbe99e06 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -9,6 +9,7 @@ fabric_db_delete, fabric_db_doc_count, fabric_db_info, + fabric_db_meta, fabric_dict, fabric_doc_attachments, fabric_doc_missing_revs, diff --git a/src/fabric.erl b/src/fabric.erl index 5d03c5fa..1be97a98 100644 --- a/src/fabric.erl +++ b/src/fabric.erl @@ -5,7 +5,8 @@ % DBs -export([all_dbs/0, all_dbs/1, create_db/1, create_db/2, delete_db/1, - delete_db/2, get_db_info/1, get_doc_count/1]). + delete_db/2, get_db_info/1, get_doc_count/1, set_revs_limit/3, + set_security/3, get_revs_limit/1, get_security/1]). % Documents -export([open_doc/3, open_revs/4, get_missing_revs/2, update_doc/3, @@ -57,6 +58,19 @@ delete_db(DbName) -> delete_db(DbName, Options) -> fabric_db_delete:go(dbname(DbName), opts(Options)). +set_revs_limit(DbName, Limit, Options) when is_integer(Limit), Limit > 0 -> + fabric_db_meta:set_revs_limit(dbname(DbName), Limit, opts(Options)). + +get_revs_limit(DbName) -> + {ok, Db} = fabric_util:get_db(dbname(DbName)), + try couch_db:get_revs_limit(Db) after catch couch_db:close(Db) end. + +set_security(DbName, SecObj, Options) -> + fabric_db_meta:set_security(dbname(DbName), SecObj, opts(Options)). + +get_security(DbName) -> + {ok, Db} = fabric_util:get_db(dbname(DbName)), + try couch_db:get_security(Db) after catch couch_db:close(Db) end. % doc operations open_doc(DbName, Id, Options) -> diff --git a/src/fabric_db_meta.erl b/src/fabric_db_meta.erl new file mode 100644 index 00000000..3b0ae109 --- /dev/null +++ b/src/fabric_db_meta.erl @@ -0,0 +1,30 @@ +-module(fabric_db_meta). + +-export([set_revs_limit/3, set_security/3]). + +-include("fabric.hrl"). +-include_lib("mem3/include/mem3.hrl"). + +set_revs_limit(DbName, Limit, Options) -> + Shards = mem3:shards(DbName), + Workers = fabric_util:submit_jobs(Shards, set_revs_limit, [Limit, Options]), + Waiting = length(Workers) - 1, + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Waiting) of + {ok, ok} -> + ok; + Error -> + Error + end. + +set_security(DbName, SecObj, Options) -> + Shards = mem3:shards(DbName), + Workers = fabric_util:submit_jobs(Shards, set_security, [SecObj, Options]), + Waiting = length(Workers) - 1, + fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Waiting). + +handle_message(ok, _, 0) -> + {stop, ok}; +handle_message(ok, _, Waiting) -> + {ok, Waiting - 1}; +handle_message(Error, _, _Waiting) -> + {error, Error}. \ No newline at end of file diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 6be7c3cc..0a25948f 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -3,7 +3,8 @@ -export([get_db_info/1, get_doc_count/1, get_update_seq/1]). -export([open_doc/3, open_revs/4, get_missing_revs/2, update_docs/3]). -export([all_docs/2, changes/3, map_view/4, reduce_view/4, group_info/2]). --export([create_db/3, delete_db/3, reset_validation_funs/1]). +-export([create_db/3, delete_db/3, reset_validation_funs/1, set_security/3, + set_revs_limit/3]). -include("fabric.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -160,6 +161,12 @@ get_doc_count(DbName) -> get_update_seq(DbName) -> with_db(DbName, [], {couch_db, get_update_seq, []}). +set_security(DbName, SecObj, Options) -> + with_db(DbName, Options, {couch_db, set_security, [SecObj]}). + +set_revs_limit(DbName, Limit, Options) -> + with_db(DbName, Options, {couch_db, set_revs_limit, [Limit]}). + open_doc(DbName, DocId, Options) -> with_db(DbName, Options, {couch_db, open_doc, [DocId, Options]}). diff --git a/src/fabric_util.erl b/src/fabric_util.erl index e928c5ff..d4c4f5e2 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -1,6 +1,7 @@ -module(fabric_util). --export([submit_jobs/3, cleanup/1, recv/4, receive_loop/4, receive_loop/6]). +-export([submit_jobs/3, cleanup/1, recv/4, receive_loop/4, receive_loop/6, + get_db/1]). -include("fabric.hrl"). -include_lib("mem3/include/mem3.hrl"). @@ -75,3 +76,14 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> after PerMsgTO -> timeout end. + +get_db(DbName) -> + Shards = mem3:shards(DbName), + case lists:partition(fun(#shard{node = N}) -> N =:= node() end, Shards) of + {[#shard{name = ShardName}|_], _} -> + % prefer node-local DBs + couch_db:open(ShardName, []); + {[], #shard{node = Node, name = ShardName}} -> + % but don't require them + rpc:call(Node, couch_db, open, [ShardName, []]) + end. -- cgit v1.2.3 From 6b798eb8b2f4edd3c6d1a02de6890f6ef1510785 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 14 Jul 2010 16:32:36 -0400 Subject: handle timeouts when using sync_reply --- src/fabric_rpc.erl | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 0a25948f..68733de6 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -250,7 +250,9 @@ view_fold(KV, OffsetReds, #view_acc{offset=nil, total_rows=Total} = Acc) -> ok -> view_fold(KV, OffsetReds, Acc#view_acc{offset=Offset}); stop -> - exit(normal) + exit(normal); + timeout -> + exit(timeout) end; view_fold(_KV, _Offset, #view_acc{limit=0} = Acc) -> % we scanned through limit+skip local rows @@ -272,13 +274,21 @@ view_fold({{Key,Id}, Value}, _Offset, Acc) -> couch_doc:to_json_obj(Doc0, []) end end, - rexi:sync_reply(#view_row{key=Key, id=Id, value=Value, doc=Doc}), - {ok, Acc#view_acc{limit=Limit-1}}. + case rexi:sync_reply(#view_row{key=Key, id=Id, value=Value, doc=Doc}) of + ok -> + {ok, Acc#view_acc{limit=Limit-1}}; + timeout -> + exit(timeout) + end. final_response(Total, nil) -> case rexi:sync_reply({total_and_offset, Total, Total}) of ok -> rexi:reply(complete); - stop -> ok end; + stop -> + ok; + timeout -> + exit(timeout) + end; final_response(_Total, _Offset) -> rexi:reply(complete). @@ -307,7 +317,9 @@ send(Key, Value, #view_acc{limit=Limit} = Acc) -> ok -> {ok, Acc#view_acc{limit=Limit-1}}; stop -> - exit(normal) + exit(normal); + timeout -> + exit(timeout) end. changes_enumerator(DocInfos, {Db, _Seq, Args}) -> -- cgit v1.2.3 From 36bd40e5e1227817ea9a6f386b44758bc8a46359 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 15 Jul 2010 14:00:27 -0400 Subject: don't badmatch if first shard in keyrange is missing --- src/fabric_view.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fabric_view.erl b/src/fabric_view.erl index 4f29abf7..d35283d7 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -15,7 +15,7 @@ is_progress_possible([]) -> is_progress_possible(Counters) -> Ranges = fabric_dict:fold(fun(#shard{range=[X,Y]}, _, A) -> [{X,Y}|A] end, [], Counters), - [{0, Tail0} | Rest] = lists:ukeysort(1, Ranges), + [{Start, Tail0} | Rest] = lists:ukeysort(1, Ranges), Result = lists:foldl(fun (_, fail) -> % we've already declared failure @@ -35,7 +35,7 @@ is_progress_possible(Counters) -> Else end end, Tail0, Rest), - Result =:= complete. + (Start =:= 0) andalso (Result =:= complete). -spec remove_overlapping_shards(#shard{}, [{#shard{}, any()}]) -> [{#shard{}, any()}]. -- cgit v1.2.3 From fca120f299df7fd78fc1681993f4745d1bfc3ce2 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Thu, 15 Jul 2010 14:28:11 -0400 Subject: update to new _changes feed format --- src/fabric_rpc.erl | 8 ++++---- src/fabric_view_changes.erl | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index 68733de6..e1bd6831 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -322,11 +322,11 @@ send(Key, Value, #view_acc{limit=Limit} = Acc) -> exit(timeout) end. -changes_enumerator(DocInfos, {Db, _Seq, Args}) -> +changes_enumerator(DocInfo, {Db, _Seq, Args}) -> #changes_args{include_docs=IncludeDocs, filter=FilterFun} = Args, - [#doc_info{id=Id, high_seq=Seq, revs=[#rev_info{deleted=Del,rev=Rev}|_]}|_] - = DocInfos, - case [Result || Result <- FilterFun(DocInfos), Result /= null] of + #doc_info{id=Id, high_seq=Seq, revs=[#rev_info{deleted=Del,rev=Rev}|_]} + = DocInfo, + case [Result || Result <- FilterFun(DocInfo), Result /= null] of [] -> {ok, {Db, Seq, Args}}; Results -> diff --git a/src/fabric_view_changes.erl b/src/fabric_view_changes.erl index 39755a01..03f362a7 100644 --- a/src/fabric_view_changes.erl +++ b/src/fabric_view_changes.erl @@ -164,8 +164,12 @@ handle_message({complete, EndSeq}, Worker, State) -> end end. -make_changes_args(Options) -> - Options. +make_changes_args(#changes_args{style=main_only, filter=undefined}=Args) -> + Args#changes_args{filter = fun couch_changes:main_only_filter/1}; +make_changes_args(#changes_args{style=all_docs, filter=undefined}=Args) -> + Args#changes_args{filter = fun couch_changes:all_docs_filter/1}; +make_changes_args(Args) -> + Args. get_start_seq(_DbName, #changes_args{dir=fwd, since=Since}) -> Since; -- cgit v1.2.3 From 7dc0964b68ed09aa1f24685883d5d93078389d1f Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 16 Jul 2010 11:42:02 -0400 Subject: fix ok response for set_security --- src/fabric_db_meta.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/fabric_db_meta.erl b/src/fabric_db_meta.erl index 3b0ae109..ee15fc72 100644 --- a/src/fabric_db_meta.erl +++ b/src/fabric_db_meta.erl @@ -20,7 +20,12 @@ set_security(DbName, SecObj, Options) -> Shards = mem3:shards(DbName), Workers = fabric_util:submit_jobs(Shards, set_security, [SecObj, Options]), Waiting = length(Workers) - 1, - fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Waiting). + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Waiting) of + {ok, ok} -> + ok; + Error -> + Error + end. handle_message(ok, _, 0) -> {stop, ok}; -- cgit v1.2.3 From a3853280ae617e21817a5097c431f1e62d3dbd47 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 16 Jul 2010 15:10:20 -0400 Subject: very stupid bug --- src/fabric_util.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fabric_util.erl b/src/fabric_util.erl index d4c4f5e2..639a32e7 100644 --- a/src/fabric_util.erl +++ b/src/fabric_util.erl @@ -83,7 +83,7 @@ get_db(DbName) -> {[#shard{name = ShardName}|_], _} -> % prefer node-local DBs couch_db:open(ShardName, []); - {[], #shard{node = Node, name = ShardName}} -> + {[], [#shard{node = Node, name = ShardName}|_]} -> % but don't require them rpc:call(Node, couch_db, open, [ShardName, []]) end. -- cgit v1.2.3 From ba55fe683f7e8d8432cc11b17c7af2e7ee46105f Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 16 Jul 2010 18:32:32 -0400 Subject: nicer error messages for missing attachment stubs --- src/fabric_doc_update.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index 0898d96c..5c4cb132 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -50,6 +50,8 @@ handle_message({ok, Replies}, Worker, Acc0) -> % no point in trying to finalize anything yet {ok, {WaitingCount - 1, DocCount, W, GroupedDocs, DocReplyDict}} end; +handle_message({missing_stub, Stub}, _, _) -> + throw({missing_stub, Stub}); handle_message({not_found, no_db_file} = X, Worker, Acc0) -> {_, _, _, GroupedDocs, _} = Acc0, Docs = couch_util:get_value(Worker, GroupedDocs), -- cgit v1.2.3 From 31b2cb626372e59aff719c5ed06652d2bbb6675a Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 26 Jul 2010 20:54:26 -0400 Subject: appups for 1.2.1 --- ebin/fabric.app | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index bbe99e06..d5f027a9 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -2,7 +2,7 @@ {application, fabric, [{description, "Routing and proxying layer for CouchDB cluster"}, - {vsn, "1.0"}, + {vsn, "1.0.1"}, {modules, [ fabric, fabric_db_create, -- cgit v1.2.3 From c2cb8bd724ad870ae347334ddc4f6862b9244514 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 27 Jul 2010 11:38:03 -0400 Subject: add registered field and fix formatting of .app file --- ebin/fabric.app | 57 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index d5f027a9..dde0393c 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -1,29 +1,28 @@ -%% fabric app resource file - -{application, fabric, - [{description, "Routing and proxying layer for CouchDB cluster"}, - {vsn, "1.0.1"}, - {modules, [ - fabric, - fabric_db_create, - fabric_db_delete, - fabric_db_doc_count, - fabric_db_info, - fabric_db_meta, - fabric_dict, - fabric_doc_attachments, - fabric_doc_missing_revs, - fabric_doc_open, - fabric_doc_open_revs, - fabric_doc_update, - fabric_group_info, - fabric_rpc, - fabric_util, - fabric_view, - fabric_view_all_docs, - fabric_view_changes, - fabric_view_map, - fabric_view_reduce - ]}, - {applications, [kernel, stdlib, couch, rexi, mem3]} - ]}. +{application, fabric, [ + {description, "Routing and proxying layer for CouchDB cluster"}, + {vsn, "1.0.1"}, + {modules, [ + fabric, + fabric_db_create, + fabric_db_delete, + fabric_db_doc_count, + fabric_db_info, + fabric_db_meta, + fabric_dict, + fabric_doc_attachments, + fabric_doc_missing_revs, + fabric_doc_open, + fabric_doc_open_revs, + fabric_doc_update, + fabric_group_info, + fabric_rpc, + fabric_util, + fabric_view, + fabric_view_all_docs, + fabric_view_changes, + fabric_view_map, + fabric_view_reduce + ]}, + {registered, []}, + {applications, [kernel, stdlib, couch, rexi, mem3]} +]}. -- cgit v1.2.3 From 33ee9064a3434baef7b5f026c8e41b563ac8d5e0 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 27 Jul 2010 11:52:55 -0400 Subject: add the appups for real --- ebin/fabric.appup | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 ebin/fabric.appup diff --git a/ebin/fabric.appup b/ebin/fabric.appup new file mode 100644 index 00000000..d1f43486 --- /dev/null +++ b/ebin/fabric.appup @@ -0,0 +1,3 @@ +{"1.0.1",[{"1.0",[ + {load_module, fabric_doc_update} +]}],[{"1.0",[]}]}. -- cgit v1.2.3 From f8c3d90e660d1f1bd4b65c7cacb6f7e748ce951e Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 28 Jul 2010 15:10:21 -0400 Subject: preliminary support for multipart/related PUTs with N>1 --- src/fabric_rpc.erl | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl index e1bd6831..f56e3f68 100644 --- a/src/fabric_rpc.erl +++ b/src/fabric_rpc.erl @@ -191,13 +191,14 @@ get_missing_revs(DbName, IdRevsList) -> Error end). -update_docs(DbName, Docs, Options) -> +update_docs(DbName, Docs0, Options) -> case proplists:get_value(replicated_changes, Options) of true -> X = replicated_changes; _ -> X = interactive_edit end, + Docs = make_att_readers(Docs0), with_db(DbName, Options, {couch_db, update_docs, [Docs, Options, X]}). group_info(DbName, Group0) -> @@ -369,3 +370,19 @@ possible_ancestors(FullInfo, MissingRevs) -> Acc end end, [], LeafRevs). + +make_att_readers([]) -> + []; +make_att_readers([#doc{atts=Atts0} = Doc | Rest]) -> + % % go through the attachments looking for 'follows' in the data, + % % replace with function that reads the data from MIME stream. + Atts = [Att#att{data=make_att_reader(D)} || #att{data=D} = Att <- Atts0], + [Doc#doc{atts = Atts} | make_att_readers(Rest)]. + +make_att_reader({follows, Parser}) -> + fun() -> + Parser ! {get_bytes, self()}, + receive {bytes, Bytes} -> Bytes end + end; +make_att_reader(Else) -> + Else. -- cgit v1.2.3 From 62f6260f8b471569288e1917d0c79104fbf898ec Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 28 Jul 2010 15:31:12 -0400 Subject: bugfix for is_progress_possible with q=1 --- src/fabric_view.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fabric_view.erl b/src/fabric_view.erl index d35283d7..49a3a55a 100644 --- a/src/fabric_view.erl +++ b/src/fabric_view.erl @@ -34,7 +34,7 @@ is_progress_possible(Counters) -> % the normal condition, adding to the tail Else end - end, Tail0, Rest), + end, if (Tail0+1) =:= (2 bsl 31) -> complete; true -> Tail0 end, Rest), (Start =:= 0) andalso (Result =:= complete). -spec remove_overlapping_shards(#shard{}, [{#shard{}, any()}]) -> -- cgit v1.2.3 From 1a37af5d8bc9fc72c6ff8e550b858f4a3c6e48ab Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Wed, 28 Jul 2010 17:35:57 -0400 Subject: appups for 1.2.2 --- ebin/fabric.app | 2 +- ebin/fabric.appup | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index dde0393c..054b2586 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -1,6 +1,6 @@ {application, fabric, [ {description, "Routing and proxying layer for CouchDB cluster"}, - {vsn, "1.0.1"}, + {vsn, "1.0.2"}, {modules, [ fabric, fabric_db_create, diff --git a/ebin/fabric.appup b/ebin/fabric.appup index d1f43486..312af372 100644 --- a/ebin/fabric.appup +++ b/ebin/fabric.appup @@ -1,3 +1,4 @@ -{"1.0.1",[{"1.0",[ - {load_module, fabric_doc_update} -]}],[{"1.0",[]}]}. +{"1.0.2",[{"1.0.1",[ + {load_module, fabric_rpc}, + {load_module, fabric_view} +]}],[{"1.0.1",[]}]}. -- cgit v1.2.3 From 3bddab0c470a2876ffbf9924958b34e79150bf69 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 30 Jul 2010 10:27:54 -0400 Subject: include a sequence sum in the _changes seq for sorting, BugzID 10588 --- src/fabric_view_changes.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/fabric_view_changes.erl b/src/fabric_view_changes.erl index 03f362a7..6030df1d 100644 --- a/src/fabric_view_changes.erl +++ b/src/fabric_view_changes.erl @@ -198,7 +198,9 @@ collect_update_seqs(Seq, Shard, Counters) when is_integer(Seq) -> pack_seqs(Workers) -> SeqList = [{N,R,S} || {#shard{node=N, range=R}, S} <- Workers], - couch_util:encodeBase64Url(term_to_binary(SeqList, [compressed])). + SeqSum = lists:sum(element(2, lists:unzip(Workers))), + Opaque = couch_util:encodeBase64Url(term_to_binary(SeqList, [compressed])), + list_to_binary([integer_to_list(SeqSum), $-, Opaque]). unpack_seqs(0, DbName) -> fabric_dict:init(mem3:shards(DbName), 0); @@ -207,11 +209,13 @@ unpack_seqs("0", DbName) -> fabric_dict:init(mem3:shards(DbName), 0); unpack_seqs(Packed, DbName) -> + {match, [Opaque]} = re:run(Packed, "^([0-9]+-)?(?.*)", [{capture, + [opaque], binary}]), % TODO relies on internal structure of fabric_dict as keylist lists:map(fun({Node, [A,B], Seq}) -> Shard = #shard{node=Node, range=[A,B], dbname=DbName}, {mem3_util:name_shard(Shard), Seq} - end, binary_to_term(couch_util:decodeBase64Url(Packed))). + end, binary_to_term(couch_util:decodeBase64Url(Opaque))). start_update_notifiers(DbName) -> lists:map(fun(#shard{node=Node, name=Name}) -> -- cgit v1.2.3 From 575d736f197a297c841bd45f930e1a424fcd1ce3 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Fri, 30 Jul 2010 15:16:56 -0400 Subject: appups for 1.2.3 --- ebin/fabric.app | 2 +- ebin/fabric.appup | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ebin/fabric.app b/ebin/fabric.app index 054b2586..8a565d8a 100644 --- a/ebin/fabric.app +++ b/ebin/fabric.app @@ -1,6 +1,6 @@ {application, fabric, [ {description, "Routing and proxying layer for CouchDB cluster"}, - {vsn, "1.0.2"}, + {vsn, "1.0.3"}, {modules, [ fabric, fabric_db_create, diff --git a/ebin/fabric.appup b/ebin/fabric.appup index 312af372..ef5dc496 100644 --- a/ebin/fabric.appup +++ b/ebin/fabric.appup @@ -1,4 +1,3 @@ -{"1.0.2",[{"1.0.1",[ - {load_module, fabric_rpc}, - {load_module, fabric_view} -]}],[{"1.0.1",[]}]}. +{"1.0.3",[{"1.0.2",[ + {load_module, fabric_view_changes} +]}],[{"1.0.2",[]}]}. -- cgit v1.2.3 From 38228cdaeefd6ad2f9ec2452e05794caa850cf84 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Mon, 9 Aug 2010 01:11:00 -0400 Subject: fix timeout when calling update_docs with empty list --- src/fabric_doc_update.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fabric_doc_update.erl b/src/fabric_doc_update.erl index 5c4cb132..f0fcf112 100644 --- a/src/fabric_doc_update.erl +++ b/src/fabric_doc_update.erl @@ -6,6 +6,8 @@ -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). +go(_, [], _) -> + {ok, []}; go(DbName, AllDocs, Opts) -> validate_atomic_update(DbName, AllDocs, lists:member(all_or_nothing, Opts)), Options = lists:delete(all_or_nothing, Opts), -- cgit v1.2.3 From 5e2f90537f5b54adc94c58b58512a05b058fa804 Mon Sep 17 00:00:00 2001 From: Adam Kocoloski Date: Tue, 10 Aug 2010 17:34:08 -0400 Subject: validate dbnames before submitting jobs --- src/fabric_db_create.erl | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/fabric_db_create.erl b/src/fabric_db_create.erl index 2fca0eed..d10bcc22 100644 --- a/src/fabric_db_create.erl +++ b/src/fabric_db_create.erl @@ -5,18 +5,25 @@ -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). +-define(DBNAME_REGEX, "^[a-z][a-z0-9\\_\\$()\\+\\-\\/\\s.]*$"). + %% @doc Create a new database, and all its partition files across the cluster %% Options is proplist with user_ctx, n, q go(DbName, Options) -> - Shards = mem3:choose_shards(DbName, Options), - Doc = make_document(Shards), - Workers = fabric_util:submit_jobs(Shards, create_db, [Options, Doc]), - Acc0 = fabric_dict:init(Workers, nil), - case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of - {ok, _} -> - ok; - Else -> - Else + case re:run(DbName, ?DBNAME_REGEX, [{capture,none}]) of + match -> + Shards = mem3:choose_shards(DbName, Options), + Doc = make_document(Shards), + Workers = fabric_util:submit_jobs(Shards, create_db, [Options, Doc]), + Acc0 = fabric_dict:init(Workers, nil), + case fabric_util:recv(Workers, #shard.ref, fun handle_message/3, Acc0) of + {ok, _} -> + ok; + Else -> + Else + end; + nomatch -> + {error, illegal_database_name} end. handle_message(Msg, Shard, Counters) -> -- cgit v1.2.3