summaryrefslogtreecommitdiff
path: root/src/couchdb/couch_util.erl
diff options
context:
space:
mode:
Diffstat (limited to 'src/couchdb/couch_util.erl')
-rw-r--r--src/couchdb/couch_util.erl307
1 files changed, 164 insertions, 143 deletions
diff --git a/src/couchdb/couch_util.erl b/src/couchdb/couch_util.erl
index 6edfb781..c11ba994 100644
--- a/src/couchdb/couch_util.erl
+++ b/src/couchdb/couch_util.erl
@@ -12,18 +12,25 @@
-module(couch_util).
--export([priv_dir/0, start_driver/1,terminate_linked/1]).
+-export([priv_dir/0, start_driver/1, normpath/1]).
-export([should_flush/0, should_flush/1, to_existing_atom/1]).
-export([rand32/0, implode/2, collate/2, collate/3]).
--export([abs_pathname/1,abs_pathname/2, trim/1, ascii_lower/1]).
--export([encodeBase64/1, decodeBase64/1, encodeBase64Url/1, decodeBase64Url/1,
- to_hex/1,parse_term/1, dict_find/3]).
--export([file_read_size/1, get_nested_json_value/2, json_user_ctx/1]).
+-export([abs_pathname/1,abs_pathname/2, trim/1]).
+-export([encodeBase64Url/1, decodeBase64Url/1]).
+-export([to_hex/1, parse_term/1, dict_find/3]).
+-export([get_nested_json_value/2, json_user_ctx/1]).
+-export([proplist_apply_field/2, json_apply_field/2]).
-export([to_binary/1, to_integer/1, to_list/1, url_encode/1]).
-export([json_encode/1, json_decode/1]).
+-export([verify/2,simple_call/2,shutdown_sync/1]).
+-export([compressible_att_type/1]).
+-export([get_value/2, get_value/3]).
+-export([md5/1, md5_init/0, md5_update/2, md5_final/1]).
+-export([reorder_results/2]).
+-export([url_strip_password/1]).
+-export([encode_doc_id/1]).
-include("couch_db.hrl").
--include_lib("kernel/include/file.hrl").
% arbitrarily chosen amount of memory to use before flushing to disk
-define(FLUSH_MAX_MEM, 10000000).
@@ -48,23 +55,57 @@ start_driver(LibDir) ->
exit(erl_ddll:format_error(Error))
end.
+% Normalize a pathname by removing .. and . components.
+normpath(Path) ->
+ normparts(filename:split(Path), []).
+
+normparts([], Acc) ->
+ filename:join(lists:reverse(Acc));
+normparts([".." | RestParts], [_Drop | RestAcc]) ->
+ normparts(RestParts, RestAcc);
+normparts(["." | RestParts], Acc) ->
+ normparts(RestParts, Acc);
+normparts([Part | RestParts], Acc) ->
+ normparts(RestParts, [Part | Acc]).
+
% works like list_to_existing_atom, except can be list or binary and it
% gives you the original value instead of an error if no existing atom.
to_existing_atom(V) when is_list(V) ->
- try list_to_existing_atom(V) catch _ -> V end;
+ try list_to_existing_atom(V) catch _:_ -> V end;
to_existing_atom(V) when is_binary(V) ->
- try list_to_existing_atom(?b2l(V)) catch _ -> V end;
+ try list_to_existing_atom(?b2l(V)) catch _:_ -> V end;
to_existing_atom(V) when is_atom(V) ->
V.
+shutdown_sync(Pid) when not is_pid(Pid)->
+ ok;
+shutdown_sync(Pid) ->
+ MRef = erlang:monitor(process, Pid),
+ try
+ catch unlink(Pid),
+ catch exit(Pid, shutdown),
+ receive
+ {'DOWN', MRef, _, _, _} ->
+ ok
+ end
+ after
+ erlang:demonitor(MRef, [flush])
+ end.
+
-terminate_linked(normal) ->
- terminate_linked(shutdown);
-terminate_linked(Reason) ->
- {links, Links} = process_info(self(), links),
- [catch exit(Pid, Reason) || Pid <- Links],
- ok.
-
+simple_call(Pid, Message) ->
+ MRef = erlang:monitor(process, Pid),
+ try
+ Pid ! {self(), Message},
+ receive
+ {Pid, Result} ->
+ Result;
+ {'DOWN', MRef, _, _, Reason} ->
+ exit(Reason)
+ end
+ after
+ erlang:demonitor(MRef, [flush])
+ end.
to_hex([]) ->
[];
@@ -83,9 +124,19 @@ parse_term(List) ->
{ok, Tokens, _} = erl_scan:string(List ++ "."),
erl_parse:parse_term(Tokens).
+get_value(Key, List) ->
+ get_value(Key, List, undefined).
+
+get_value(Key, List, Default) ->
+ case lists:keysearch(Key, 1, List) of
+ {value, {Key,Value}} ->
+ Value;
+ false ->
+ Default
+ end.
get_nested_json_value({Props}, [Key|Keys]) ->
- case proplists:get_value(Key, Props, nil) of
+ case couch_util:get_value(Key, Props, nil) of
nil -> throw({not_found, <<"missing json key: ", Key/binary>>});
Value -> get_nested_json_value(Value, Keys)
end;
@@ -94,6 +145,19 @@ get_nested_json_value(Value, []) ->
get_nested_json_value(_NotJSONObj, _) ->
throw({not_found, json_mismatch}).
+proplist_apply_field(H, L) ->
+ {R} = json_apply_field(H, {L}),
+ R.
+
+json_apply_field(H, {L}) ->
+ json_apply_field(H, L, []).
+json_apply_field({Key, NewValue}, [{Key, _OldVal} | Headers], Acc) ->
+ json_apply_field({Key, NewValue}, Headers, Acc);
+json_apply_field({Key, NewValue}, [{OtherKey, OtherVal} | Headers], Acc) ->
+ json_apply_field({Key, NewValue}, Headers, [{OtherKey, OtherVal} | Acc]);
+json_apply_field({Key, NewValue}, [], Acc) ->
+ {[{Key, NewValue}|Acc]}.
+
json_user_ctx(#db{name=DbName, user_ctx=Ctx}) ->
{[{<<"db">>, DbName},
{<<"name">>,Ctx#user_ctx.name},
@@ -140,18 +204,6 @@ separate_cmd_args(" " ++ Rest, CmdAcc) ->
separate_cmd_args([Char|Rest], CmdAcc) ->
separate_cmd_args(Rest, [Char | CmdAcc]).
-% lowercases string bytes that are the ascii characters A-Z.
-% All other characters/bytes are ignored.
-ascii_lower(String) ->
- ascii_lower(String, []).
-
-ascii_lower([], Acc) ->
- lists:reverse(Acc);
-ascii_lower([Char | RestString], Acc) when Char >= $A, Char =< $B ->
- ascii_lower(RestString, [Char + ($a-$A) | Acc]);
-ascii_lower([Char | RestString], Acc) ->
- ascii_lower(RestString, [Char | Acc]).
-
% Is a character whitespace?
is_whitespace($\s) -> true;
is_whitespace($\t) -> true;
@@ -230,114 +282,18 @@ should_flush(MemThreshHold) ->
ProcMem2+BinMem2 > MemThreshHold;
true -> false end.
+encodeBase64Url(Url) ->
+ Url1 = iolist_to_binary(re:replace(base64:encode(Url), "=+$", "")),
+ Url2 = iolist_to_binary(re:replace(Url1, "/", "_", [global])),
+ iolist_to_binary(re:replace(Url2, "\\+", "-", [global])).
-%%% Purpose : Base 64 encoding and decoding.
-%%% Copied from ssl_base_64 to avoid using the
-%%% erlang ssl library
-
--define(st(X,A), ((X-A+256) div 256)).
-
-%% A PEM encoding consists of characters A-Z, a-z, 0-9, +, / and
-%% =. Each character encodes a 6 bits value from 0 to 63 (A = 0, / =
-%% 63); = is a padding character.
-%%
-
-%%
-%% encode64(Bytes|Binary) -> binary
-%%
-%% Take 3 bytes a time (3 x 8 = 24 bits), and make 4 characters out of
-%% them (4 x 6 = 24 bits).
-%%
-encodeBase64(Bs) when is_list(Bs) ->
- encodeBase64(iolist_to_binary(Bs), <<>>);
-encodeBase64(Bs) ->
- encodeBase64(Bs, <<>>).
-
-encodeBase64(<<B:3/binary, Bs/binary>>, Acc) ->
- <<C1:6, C2:6, C3:6, C4:6>> = B,
- encodeBase64(Bs, <<Acc/binary, (enc(C1)), (enc(C2)), (enc(C3)), (enc(C4))>>);
-encodeBase64(<<B:2/binary>>, Acc) ->
- <<C1:6, C2:6, C3:6, _:6>> = <<B/binary, 0>>,
- <<Acc/binary, (enc(C1)), (enc(C2)), (enc(C3)), $=>>;
-encodeBase64(<<B:1/binary>>, Acc) ->
- <<C1:6, C2:6, _:12>> = <<B/binary, 0, 0>>,
- <<Acc/binary, (enc(C1)), (enc(C2)), $=, $=>>;
-encodeBase64(<<>>, Acc) ->
- Acc.
-
-encodeBase64Url(Bs) when is_list(Bs) ->
- encodeBase64Url(list_to_binary(Bs), <<>>);
-encodeBase64Url(Bs) ->
- encodeBase64Url(Bs, <<>>).
-
-encodeBase64Url(<<B:3/binary, Bs/binary>>, Acc) ->
- <<C1:6, C2:6, C3:6, C4:6>> = B,
- encodeBase64Url(Bs, <<Acc/binary, (encUrl(C1)), (encUrl(C2)), (encUrl(C3)), (encUrl(C4))>>);
-encodeBase64Url(<<B:2/binary>>, Acc) ->
- <<C1:6, C2:6, C3:6, _:6>> = <<B/binary, 0>>,
- <<Acc/binary, (encUrl(C1)), (encUrl(C2)), (encUrl(C3))>>;
-encodeBase64Url(<<B:1/binary>>, Acc) ->
- <<C1:6, C2:6, _:12>> = <<B/binary, 0, 0>>,
- <<Acc/binary, (encUrl(C1)), (encUrl(C2))>>;
-encodeBase64Url(<<>>, Acc) ->
- Acc.
-
-%%
-%% decodeBase64(BinaryChars) -> Binary
-%%
-decodeBase64(Cs) when is_list(Cs) ->
- decodeBase64(list_to_binary(Cs));
-decodeBase64(Cs) ->
- decode1(Cs, <<>>).
-
-decode1(<<C1, C2, $=, $=>>, Acc) ->
- <<B1, _:16>> = <<(dec(C1)):6, (dec(C2)):6, 0:12>>,
- <<Acc/binary, B1>>;
-decode1(<<C1, C2, C3, $=>>, Acc) ->
- <<B1, B2, _:8>> = <<(dec(C1)):6, (dec(C2)):6, (dec(C3)):6, (dec(0)):6>>,
- <<Acc/binary, B1, B2>>;
-decode1(<<C1, C2, C3, C4, Cs/binary>>, Acc) ->
- Bin = <<Acc/binary, (dec(C1)):6, (dec(C2)):6, (dec(C3)):6, (dec(C4)):6>>,
- decode1(Cs, Bin);
-decode1(<<>>, Acc) ->
- Acc.
-
-decodeBase64Url(Cs) when is_list(Cs) ->
- decodeBase64Url(list_to_binary(Cs));
-decodeBase64Url(Cs) ->
- decode1Url(Cs, <<>>).
-
-decode1Url(<<C1, C2>>, Acc) ->
- <<B1, _:16>> = <<(decUrl(C1)):6, (decUrl(C2)):6, 0:12>>,
- <<Acc/binary, B1>>;
-decode1Url(<<C1, C2, C3>>, Acc) ->
- <<B1, B2, _:8>> = <<(decUrl(C1)):6, (decUrl(C2)):6, (decUrl(C3)):6, (decUrl(0)):6>>,
- <<Acc/binary, B1, B2>>;
-decode1Url(<<C1, C2, C3, C4, Cs/binary>>, Acc) ->
- Bin = <<Acc/binary, (decUrl(C1)):6, (decUrl(C2)):6, (decUrl(C3)):6, (decUrl(C4)):6>>,
- decode1Url(Cs, Bin);
-decode1Url(<<>>, Acc) ->
- Acc.
-
-%% enc/1 and dec/1
-%%
-%% Mapping: 0-25 -> A-Z, 26-51 -> a-z, 52-61 -> 0-9, 62 -> +, 63 -> /
-%%
-enc(C) ->
- 65 + C + 6*?st(C,26) - 75*?st(C,52) -15*?st(C,62) + 3*?st(C,63).
-
-dec(C) ->
- 62*?st(C,43) + ?st(C,47) + (C-59)*?st(C,48) - 69*?st(C,65) - 6*?st(C,97).
-
-%% encUrl/1 and decUrl/1
-%%
-%% Mapping: 0-25 -> A-Z, 26-51 -> a-z, 52-61 -> 0-9, 62 -> -, 63 -> _
-%%
-encUrl(C) ->
- 65 + C + 6*?st(C,26) - 75*?st(C,52) -13*?st(C,62) + 49*?st(C,63).
-
-decUrl(C) ->
- 62*?st(C,45) + (C-58)*?st(C,48) - 69*?st(C,65) + 33*?st(C,95) - 39*?st(C,97).
+decodeBase64Url(Url64) ->
+ Url1 = re:replace(iolist_to_binary(Url64), "-", "+", [global]),
+ Url2 = iolist_to_binary(
+ re:replace(iolist_to_binary(Url1), "_", "/", [global])
+ ),
+ Padding = ?l2b(lists:duplicate((4 - size(Url2) rem 4) rem 4, $=)),
+ base64:decode(<<Url2/binary, Padding/binary>>).
dict_find(Key, Dict, DefaultValue) ->
case dict:find(Key, Dict) of
@@ -347,14 +303,6 @@ dict_find(Key, Dict, DefaultValue) ->
DefaultValue
end.
-
-file_read_size(FileName) ->
- case file:read_file_info(FileName) of
- {ok, FileInfo} ->
- FileInfo#file_info.size;
- Error -> Error
- end.
-
to_binary(V) when is_binary(V) ->
V;
to_binary(V) when is_list(V) ->
@@ -423,3 +371,76 @@ json_decode(V) ->
_Type:_Error ->
throw({invalid_json,V})
end.
+
+verify([X|RestX], [Y|RestY], Result) ->
+ verify(RestX, RestY, (X bxor Y) bor Result);
+verify([], [], Result) ->
+ Result == 0.
+
+verify(<<X/binary>>, <<Y/binary>>) ->
+ verify(?b2l(X), ?b2l(Y));
+verify(X, Y) when is_list(X) and is_list(Y) ->
+ case length(X) == length(Y) of
+ true ->
+ verify(X, Y, 0);
+ false ->
+ false
+ end;
+verify(_X, _Y) -> false.
+
+compressible_att_type(MimeType) when is_binary(MimeType) ->
+ compressible_att_type(?b2l(MimeType));
+compressible_att_type(MimeType) ->
+ TypeExpList = re:split(
+ couch_config:get("attachments", "compressible_types", ""),
+ ", ?",
+ [{return, list}]
+ ),
+ lists:any(
+ fun(TypeExp) ->
+ Regexp = ["^\\s*", re:replace(TypeExp, "\\*", ".*"), "\\s*$"],
+ re:run(MimeType, Regexp, [caseless]) =/= nomatch
+ end,
+ [T || T <- TypeExpList, T /= []]
+ ).
+
+-spec md5(Data::(iolist() | binary())) -> Digest::binary().
+md5(Data) ->
+ try crypto:md5(Data) catch error:_ -> erlang:md5(Data) end.
+
+-spec md5_init() -> Context::binary().
+md5_init() ->
+ try crypto:md5_init() catch error:_ -> erlang:md5_init() end.
+
+-spec md5_update(Context::binary(), Data::(iolist() | binary())) ->
+ NewContext::binary().
+md5_update(Ctx, D) ->
+ try crypto:md5_update(Ctx,D) catch error:_ -> erlang:md5_update(Ctx,D) end.
+
+-spec md5_final(Context::binary()) -> Digest::binary().
+md5_final(Ctx) ->
+ try crypto:md5_final(Ctx) catch error:_ -> erlang:md5_final(Ctx) end.
+
+% linear search is faster for small lists, length() is 0.5 ms for 100k list
+reorder_results(Keys, SortedResults) when length(Keys) < 100 ->
+ [couch_util:get_value(Key, SortedResults) || Key <- Keys];
+reorder_results(Keys, SortedResults) ->
+ KeyDict = dict:from_list(SortedResults),
+ [dict:fetch(Key, KeyDict) || Key <- Keys].
+
+url_strip_password(Url) ->
+ re:replace(Url,
+ "http(s)?://([^:]+):[^@]+@(.*)$",
+ "http\\1://\\2:*****@\\3",
+ [{return, list}]).
+
+encode_doc_id(#doc{id = Id}) ->
+ encode_doc_id(Id);
+encode_doc_id(Id) when is_list(Id) ->
+ encode_doc_id(?l2b(Id));
+encode_doc_id(<<"_design/", Rest/binary>>) ->
+ "_design/" ++ url_encode(Rest);
+encode_doc_id(<<"_local/", Rest/binary>>) ->
+ "_local/" ++ url_encode(Rest);
+encode_doc_id(Id) ->
+ url_encode(Id).