diff options
Diffstat (limited to 'src/couchdb/couch_util.erl')
| -rw-r--r-- | src/couchdb/couch_util.erl | 307 |
1 files changed, 164 insertions, 143 deletions
diff --git a/src/couchdb/couch_util.erl b/src/couchdb/couch_util.erl index 6edfb781..c11ba994 100644 --- a/src/couchdb/couch_util.erl +++ b/src/couchdb/couch_util.erl @@ -12,18 +12,25 @@ -module(couch_util). --export([priv_dir/0, start_driver/1,terminate_linked/1]). +-export([priv_dir/0, start_driver/1, normpath/1]). -export([should_flush/0, should_flush/1, to_existing_atom/1]). -export([rand32/0, implode/2, collate/2, collate/3]). --export([abs_pathname/1,abs_pathname/2, trim/1, ascii_lower/1]). --export([encodeBase64/1, decodeBase64/1, encodeBase64Url/1, decodeBase64Url/1, - to_hex/1,parse_term/1, dict_find/3]). --export([file_read_size/1, get_nested_json_value/2, json_user_ctx/1]). +-export([abs_pathname/1,abs_pathname/2, trim/1]). +-export([encodeBase64Url/1, decodeBase64Url/1]). +-export([to_hex/1, parse_term/1, dict_find/3]). +-export([get_nested_json_value/2, json_user_ctx/1]). +-export([proplist_apply_field/2, json_apply_field/2]). -export([to_binary/1, to_integer/1, to_list/1, url_encode/1]). -export([json_encode/1, json_decode/1]). +-export([verify/2,simple_call/2,shutdown_sync/1]). +-export([compressible_att_type/1]). +-export([get_value/2, get_value/3]). +-export([md5/1, md5_init/0, md5_update/2, md5_final/1]). +-export([reorder_results/2]). +-export([url_strip_password/1]). +-export([encode_doc_id/1]). -include("couch_db.hrl"). --include_lib("kernel/include/file.hrl"). % arbitrarily chosen amount of memory to use before flushing to disk -define(FLUSH_MAX_MEM, 10000000). @@ -48,23 +55,57 @@ start_driver(LibDir) -> exit(erl_ddll:format_error(Error)) end. +% Normalize a pathname by removing .. and . components. +normpath(Path) -> + normparts(filename:split(Path), []). + +normparts([], Acc) -> + filename:join(lists:reverse(Acc)); +normparts([".." | RestParts], [_Drop | RestAcc]) -> + normparts(RestParts, RestAcc); +normparts(["." | RestParts], Acc) -> + normparts(RestParts, Acc); +normparts([Part | RestParts], Acc) -> + normparts(RestParts, [Part | Acc]). + % works like list_to_existing_atom, except can be list or binary and it % gives you the original value instead of an error if no existing atom. to_existing_atom(V) when is_list(V) -> - try list_to_existing_atom(V) catch _ -> V end; + try list_to_existing_atom(V) catch _:_ -> V end; to_existing_atom(V) when is_binary(V) -> - try list_to_existing_atom(?b2l(V)) catch _ -> V end; + try list_to_existing_atom(?b2l(V)) catch _:_ -> V end; to_existing_atom(V) when is_atom(V) -> V. +shutdown_sync(Pid) when not is_pid(Pid)-> + ok; +shutdown_sync(Pid) -> + MRef = erlang:monitor(process, Pid), + try + catch unlink(Pid), + catch exit(Pid, shutdown), + receive + {'DOWN', MRef, _, _, _} -> + ok + end + after + erlang:demonitor(MRef, [flush]) + end. + -terminate_linked(normal) -> - terminate_linked(shutdown); -terminate_linked(Reason) -> - {links, Links} = process_info(self(), links), - [catch exit(Pid, Reason) || Pid <- Links], - ok. - +simple_call(Pid, Message) -> + MRef = erlang:monitor(process, Pid), + try + Pid ! {self(), Message}, + receive + {Pid, Result} -> + Result; + {'DOWN', MRef, _, _, Reason} -> + exit(Reason) + end + after + erlang:demonitor(MRef, [flush]) + end. to_hex([]) -> []; @@ -83,9 +124,19 @@ parse_term(List) -> {ok, Tokens, _} = erl_scan:string(List ++ "."), erl_parse:parse_term(Tokens). +get_value(Key, List) -> + get_value(Key, List, undefined). + +get_value(Key, List, Default) -> + case lists:keysearch(Key, 1, List) of + {value, {Key,Value}} -> + Value; + false -> + Default + end. get_nested_json_value({Props}, [Key|Keys]) -> - case proplists:get_value(Key, Props, nil) of + case couch_util:get_value(Key, Props, nil) of nil -> throw({not_found, <<"missing json key: ", Key/binary>>}); Value -> get_nested_json_value(Value, Keys) end; @@ -94,6 +145,19 @@ get_nested_json_value(Value, []) -> get_nested_json_value(_NotJSONObj, _) -> throw({not_found, json_mismatch}). +proplist_apply_field(H, L) -> + {R} = json_apply_field(H, {L}), + R. + +json_apply_field(H, {L}) -> + json_apply_field(H, L, []). +json_apply_field({Key, NewValue}, [{Key, _OldVal} | Headers], Acc) -> + json_apply_field({Key, NewValue}, Headers, Acc); +json_apply_field({Key, NewValue}, [{OtherKey, OtherVal} | Headers], Acc) -> + json_apply_field({Key, NewValue}, Headers, [{OtherKey, OtherVal} | Acc]); +json_apply_field({Key, NewValue}, [], Acc) -> + {[{Key, NewValue}|Acc]}. + json_user_ctx(#db{name=DbName, user_ctx=Ctx}) -> {[{<<"db">>, DbName}, {<<"name">>,Ctx#user_ctx.name}, @@ -140,18 +204,6 @@ separate_cmd_args(" " ++ Rest, CmdAcc) -> separate_cmd_args([Char|Rest], CmdAcc) -> separate_cmd_args(Rest, [Char | CmdAcc]). -% lowercases string bytes that are the ascii characters A-Z. -% All other characters/bytes are ignored. -ascii_lower(String) -> - ascii_lower(String, []). - -ascii_lower([], Acc) -> - lists:reverse(Acc); -ascii_lower([Char | RestString], Acc) when Char >= $A, Char =< $B -> - ascii_lower(RestString, [Char + ($a-$A) | Acc]); -ascii_lower([Char | RestString], Acc) -> - ascii_lower(RestString, [Char | Acc]). - % Is a character whitespace? is_whitespace($\s) -> true; is_whitespace($\t) -> true; @@ -230,114 +282,18 @@ should_flush(MemThreshHold) -> ProcMem2+BinMem2 > MemThreshHold; true -> false end. +encodeBase64Url(Url) -> + Url1 = iolist_to_binary(re:replace(base64:encode(Url), "=+$", "")), + Url2 = iolist_to_binary(re:replace(Url1, "/", "_", [global])), + iolist_to_binary(re:replace(Url2, "\\+", "-", [global])). -%%% Purpose : Base 64 encoding and decoding. -%%% Copied from ssl_base_64 to avoid using the -%%% erlang ssl library - --define(st(X,A), ((X-A+256) div 256)). - -%% A PEM encoding consists of characters A-Z, a-z, 0-9, +, / and -%% =. Each character encodes a 6 bits value from 0 to 63 (A = 0, / = -%% 63); = is a padding character. -%% - -%% -%% encode64(Bytes|Binary) -> binary -%% -%% Take 3 bytes a time (3 x 8 = 24 bits), and make 4 characters out of -%% them (4 x 6 = 24 bits). -%% -encodeBase64(Bs) when is_list(Bs) -> - encodeBase64(iolist_to_binary(Bs), <<>>); -encodeBase64(Bs) -> - encodeBase64(Bs, <<>>). - -encodeBase64(<<B:3/binary, Bs/binary>>, Acc) -> - <<C1:6, C2:6, C3:6, C4:6>> = B, - encodeBase64(Bs, <<Acc/binary, (enc(C1)), (enc(C2)), (enc(C3)), (enc(C4))>>); -encodeBase64(<<B:2/binary>>, Acc) -> - <<C1:6, C2:6, C3:6, _:6>> = <<B/binary, 0>>, - <<Acc/binary, (enc(C1)), (enc(C2)), (enc(C3)), $=>>; -encodeBase64(<<B:1/binary>>, Acc) -> - <<C1:6, C2:6, _:12>> = <<B/binary, 0, 0>>, - <<Acc/binary, (enc(C1)), (enc(C2)), $=, $=>>; -encodeBase64(<<>>, Acc) -> - Acc. - -encodeBase64Url(Bs) when is_list(Bs) -> - encodeBase64Url(list_to_binary(Bs), <<>>); -encodeBase64Url(Bs) -> - encodeBase64Url(Bs, <<>>). - -encodeBase64Url(<<B:3/binary, Bs/binary>>, Acc) -> - <<C1:6, C2:6, C3:6, C4:6>> = B, - encodeBase64Url(Bs, <<Acc/binary, (encUrl(C1)), (encUrl(C2)), (encUrl(C3)), (encUrl(C4))>>); -encodeBase64Url(<<B:2/binary>>, Acc) -> - <<C1:6, C2:6, C3:6, _:6>> = <<B/binary, 0>>, - <<Acc/binary, (encUrl(C1)), (encUrl(C2)), (encUrl(C3))>>; -encodeBase64Url(<<B:1/binary>>, Acc) -> - <<C1:6, C2:6, _:12>> = <<B/binary, 0, 0>>, - <<Acc/binary, (encUrl(C1)), (encUrl(C2))>>; -encodeBase64Url(<<>>, Acc) -> - Acc. - -%% -%% decodeBase64(BinaryChars) -> Binary -%% -decodeBase64(Cs) when is_list(Cs) -> - decodeBase64(list_to_binary(Cs)); -decodeBase64(Cs) -> - decode1(Cs, <<>>). - -decode1(<<C1, C2, $=, $=>>, Acc) -> - <<B1, _:16>> = <<(dec(C1)):6, (dec(C2)):6, 0:12>>, - <<Acc/binary, B1>>; -decode1(<<C1, C2, C3, $=>>, Acc) -> - <<B1, B2, _:8>> = <<(dec(C1)):6, (dec(C2)):6, (dec(C3)):6, (dec(0)):6>>, - <<Acc/binary, B1, B2>>; -decode1(<<C1, C2, C3, C4, Cs/binary>>, Acc) -> - Bin = <<Acc/binary, (dec(C1)):6, (dec(C2)):6, (dec(C3)):6, (dec(C4)):6>>, - decode1(Cs, Bin); -decode1(<<>>, Acc) -> - Acc. - -decodeBase64Url(Cs) when is_list(Cs) -> - decodeBase64Url(list_to_binary(Cs)); -decodeBase64Url(Cs) -> - decode1Url(Cs, <<>>). - -decode1Url(<<C1, C2>>, Acc) -> - <<B1, _:16>> = <<(decUrl(C1)):6, (decUrl(C2)):6, 0:12>>, - <<Acc/binary, B1>>; -decode1Url(<<C1, C2, C3>>, Acc) -> - <<B1, B2, _:8>> = <<(decUrl(C1)):6, (decUrl(C2)):6, (decUrl(C3)):6, (decUrl(0)):6>>, - <<Acc/binary, B1, B2>>; -decode1Url(<<C1, C2, C3, C4, Cs/binary>>, Acc) -> - Bin = <<Acc/binary, (decUrl(C1)):6, (decUrl(C2)):6, (decUrl(C3)):6, (decUrl(C4)):6>>, - decode1Url(Cs, Bin); -decode1Url(<<>>, Acc) -> - Acc. - -%% enc/1 and dec/1 -%% -%% Mapping: 0-25 -> A-Z, 26-51 -> a-z, 52-61 -> 0-9, 62 -> +, 63 -> / -%% -enc(C) -> - 65 + C + 6*?st(C,26) - 75*?st(C,52) -15*?st(C,62) + 3*?st(C,63). - -dec(C) -> - 62*?st(C,43) + ?st(C,47) + (C-59)*?st(C,48) - 69*?st(C,65) - 6*?st(C,97). - -%% encUrl/1 and decUrl/1 -%% -%% Mapping: 0-25 -> A-Z, 26-51 -> a-z, 52-61 -> 0-9, 62 -> -, 63 -> _ -%% -encUrl(C) -> - 65 + C + 6*?st(C,26) - 75*?st(C,52) -13*?st(C,62) + 49*?st(C,63). - -decUrl(C) -> - 62*?st(C,45) + (C-58)*?st(C,48) - 69*?st(C,65) + 33*?st(C,95) - 39*?st(C,97). +decodeBase64Url(Url64) -> + Url1 = re:replace(iolist_to_binary(Url64), "-", "+", [global]), + Url2 = iolist_to_binary( + re:replace(iolist_to_binary(Url1), "_", "/", [global]) + ), + Padding = ?l2b(lists:duplicate((4 - size(Url2) rem 4) rem 4, $=)), + base64:decode(<<Url2/binary, Padding/binary>>). dict_find(Key, Dict, DefaultValue) -> case dict:find(Key, Dict) of @@ -347,14 +303,6 @@ dict_find(Key, Dict, DefaultValue) -> DefaultValue end. - -file_read_size(FileName) -> - case file:read_file_info(FileName) of - {ok, FileInfo} -> - FileInfo#file_info.size; - Error -> Error - end. - to_binary(V) when is_binary(V) -> V; to_binary(V) when is_list(V) -> @@ -423,3 +371,76 @@ json_decode(V) -> _Type:_Error -> throw({invalid_json,V}) end. + +verify([X|RestX], [Y|RestY], Result) -> + verify(RestX, RestY, (X bxor Y) bor Result); +verify([], [], Result) -> + Result == 0. + +verify(<<X/binary>>, <<Y/binary>>) -> + verify(?b2l(X), ?b2l(Y)); +verify(X, Y) when is_list(X) and is_list(Y) -> + case length(X) == length(Y) of + true -> + verify(X, Y, 0); + false -> + false + end; +verify(_X, _Y) -> false. + +compressible_att_type(MimeType) when is_binary(MimeType) -> + compressible_att_type(?b2l(MimeType)); +compressible_att_type(MimeType) -> + TypeExpList = re:split( + couch_config:get("attachments", "compressible_types", ""), + ", ?", + [{return, list}] + ), + lists:any( + fun(TypeExp) -> + Regexp = ["^\\s*", re:replace(TypeExp, "\\*", ".*"), "\\s*$"], + re:run(MimeType, Regexp, [caseless]) =/= nomatch + end, + [T || T <- TypeExpList, T /= []] + ). + +-spec md5(Data::(iolist() | binary())) -> Digest::binary(). +md5(Data) -> + try crypto:md5(Data) catch error:_ -> erlang:md5(Data) end. + +-spec md5_init() -> Context::binary(). +md5_init() -> + try crypto:md5_init() catch error:_ -> erlang:md5_init() end. + +-spec md5_update(Context::binary(), Data::(iolist() | binary())) -> + NewContext::binary(). +md5_update(Ctx, D) -> + try crypto:md5_update(Ctx,D) catch error:_ -> erlang:md5_update(Ctx,D) end. + +-spec md5_final(Context::binary()) -> Digest::binary(). +md5_final(Ctx) -> + try crypto:md5_final(Ctx) catch error:_ -> erlang:md5_final(Ctx) end. + +% linear search is faster for small lists, length() is 0.5 ms for 100k list +reorder_results(Keys, SortedResults) when length(Keys) < 100 -> + [couch_util:get_value(Key, SortedResults) || Key <- Keys]; +reorder_results(Keys, SortedResults) -> + KeyDict = dict:from_list(SortedResults), + [dict:fetch(Key, KeyDict) || Key <- Keys]. + +url_strip_password(Url) -> + re:replace(Url, + "http(s)?://([^:]+):[^@]+@(.*)$", + "http\\1://\\2:*****@\\3", + [{return, list}]). + +encode_doc_id(#doc{id = Id}) -> + encode_doc_id(Id); +encode_doc_id(Id) when is_list(Id) -> + encode_doc_id(?l2b(Id)); +encode_doc_id(<<"_design/", Rest/binary>>) -> + "_design/" ++ url_encode(Rest); +encode_doc_id(<<"_local/", Rest/binary>>) -> + "_local/" ++ url_encode(Rest); +encode_doc_id(Id) -> + url_encode(Id). |