refactor att compression to allow more encodings. thanks fdmanana. COUCHDB-710

git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@934475 13f79535-47bb-0310-9956-ffa450edef68

1 files changed, 48 insertions, 43 deletions

diff --git a/src/couchdb/couch_doc.erl b/src/couchdb/couch_doc.erl
index 71d21dda..87602c23 100644
--- a/src/couchdb/couch_doc.erl
+++ b/src/couchdb/couch_doc.erl
@@ -13,7 +13,7 @@
 -module(couch_doc).
 
 -export([to_doc_info/1,to_doc_info_path/1,parse_rev/1,parse_revs/1,rev_to_str/1,revs_to_strs/1]).
--export([att_foldl/3,att_foldl_unzip/3,get_validate_doc_fun/1]).
+-export([att_foldl/3,att_foldl_decode/3,get_validate_doc_fun/1]).
 -export([from_json_obj/1,to_json_obj/2,has_stubs/1, merge_stubs/2]).
 -export([validate_docid/1]).
 -export([doc_from_multi_part_stream/2]).
@@ -85,14 +85,14 @@ to_json_attachments(Attachments, Options) ->
         Attachments,
         RevPos,
         lists:member(follows, Options),
-        lists:member(att_gzip_length, Options)
+        lists:member(att_encoding_info, Options)
     ).
 
-to_json_attachments([], _RevPosIncludeAfter, _DataToFollow, _ShowGzipLen) ->
+to_json_attachments([], _RevPosIncludeAfter, _DataToFollow, _ShowEncInfo) ->
     [];
-to_json_attachments(Atts, RevPosIncludeAfter, DataToFollow, ShowGzipLen) ->
+to_json_attachments(Atts, RevPosIncludeAfter, DataToFollow, ShowEncInfo) ->
     AttProps = lists:map(
-        fun(#att{disk_len=DiskLen, att_len=AttLen, comp=Comp}=Att) ->
+        fun(#att{disk_len=DiskLen, att_len=AttLen, encoding=Enc}=Att) ->
             {Att#att.name, {[
                 {<<"content_type">>, Att#att.type},
                 {<<"revpos">>, Att#att.revpos}
@@ -101,10 +101,10 @@ to_json_attachments(Atts, RevPosIncludeAfter, DataToFollow, ShowGzipLen) ->
                     if DataToFollow ->
                         [{<<"length">>, DiskLen}, {<<"follows">>, true}];
                     true ->
-                        AttData = case Comp of
-                        true ->
+                        AttData = case Enc of
+                        gzip ->
                             zlib:gunzip(att_to_bin(Att));
-                        _ ->
+                        identity ->
                             att_to_bin(Att)
                         end,
                         [{<<"data">>, base64:encode(AttData)}]
@@ -112,11 +112,16 @@ to_json_attachments(Atts, RevPosIncludeAfter, DataToFollow, ShowGzipLen) ->
                 true ->
                     [{<<"length">>, DiskLen}, {<<"stub">>, true}]
                 end ++
-                    case {ShowGzipLen, Comp} of
-                    {true, true} ->
-                        [{<<"gzip_length">>, AttLen}];
-                    _ ->
-                        []
+                    case {ShowEncInfo, Enc} of
+                    {false, _} ->
+                        [];
+                    {true, identity} ->
+                        [];
+                    {true, _} ->
+                        [
+                            {<<"encoding">>, couch_util:to_binary(Enc)},
+                            {<<"encoded_length">>, AttLen}
+                        ]
                     end
             }}
         end, Atts),
@@ -202,18 +207,20 @@ transfer_fields([{<<"_attachments">>, {JsonBins}} | Rest], Doc) ->
         true ->
             Type = proplists:get_value(<<"content_type">>, BinProps),
             RevPos = proplists:get_value(<<"revpos">>, BinProps, 0),
-            {AttLen, DiskLen, Comp} = att_lengths(BinProps),
-            #att{name=Name, data=stub, type=Type, att_len=AttLen,
-                disk_len=DiskLen, comp=Comp, revpos=RevPos};
+            DiskLen = proplists:get_value(<<"length">>, BinProps),
+            {Enc, EncLen} = att_encoding_info(BinProps),
+            #att{name=Name, data=stub, type=Type, att_len=EncLen,
+                disk_len=DiskLen, encoding=Enc, revpos=RevPos};
         _ ->
             Type = proplists:get_value(<<"content_type">>, BinProps,
                     ?DEFAULT_ATTACHMENT_CONTENT_TYPE),
             RevPos = proplists:get_value(<<"revpos">>, BinProps, 0),
             case proplists:get_value(<<"follows">>, BinProps) of
             true ->
-                {AttLen, DiskLen, Comp} = att_lengths(BinProps),
-                #att{name=Name, data=follows, type=Type, comp=Comp,
-                    att_len=AttLen, disk_len=DiskLen, revpos=RevPos};
+                DiskLen = proplists:get_value(<<"length">>, BinProps),
+                {Enc, EncLen} = att_encoding_info(BinProps),
+                #att{name=Name, data=follows, type=Type, encoding=Enc,
+                    att_len=EncLen, disk_len=DiskLen, revpos=RevPos};
             _ ->
                 Value = proplists:get_value(<<"data">>, BinProps),
                 Bin = base64:decode(Value),
@@ -261,14 +268,14 @@ transfer_fields([{<<"_",Name/binary>>, _} | _], _) ->
 transfer_fields([Field | Rest], #doc{body=Fields}=Doc) ->
     transfer_fields(Rest, Doc#doc{body=[Field|Fields]}).
 
-att_lengths(BinProps) ->
+att_encoding_info(BinProps) ->
     DiskLen = proplists:get_value(<<"length">>, BinProps),
-    GzipLen = proplists:get_value(<<"gzip_length">>, BinProps),
-    case GzipLen of
+    case proplists:get_value(<<"encoding">>, BinProps) of
     undefined ->
-        {DiskLen, DiskLen, false};
-    _ ->
-        {GzipLen, DiskLen, true}
+        {identity, DiskLen};
+    Enc ->
+        EncodedLen = proplists:get_value(<<"encoded_length">>, BinProps, DiskLen),
+        {list_to_atom(?b2l(Enc)), EncodedLen}
     end.
 
 to_doc_info(FullDocInfo) ->
@@ -308,8 +315,8 @@ att_foldl(#att{data={Fd,Sp},md5=Md5}, Fun, Acc) ->
 att_foldl(#att{data=DataFun,att_len=Len}, Fun, Acc) when is_function(DataFun) ->
    fold_streamed_data(DataFun, Len, Fun, Acc).
 
-att_foldl_unzip(#att{data={Fd,Sp},md5=Md5}, Fun, Acc) ->
-    couch_stream:foldl_unzip(Fd, Sp, Md5, Fun, Acc).
+att_foldl_decode(#att{data={Fd,Sp},md5=Md5,encoding=Enc}, Fun, Acc) ->
+    couch_stream:foldl_decode(Fd, Sp, Md5, Enc, Fun, Acc).
 
 att_to_bin(#att{data=Bin}) when is_binary(Bin) ->
     Bin;
@@ -377,7 +384,7 @@ fold_streamed_data(RcvFun, LenLeft, Fun, Acc) when LenLeft > 0->
     fold_streamed_data(RcvFun, LenLeft - size(Bin), Fun, ResultAcc).
 
 len_doc_to_multi_part_stream(Boundary, JsonBytes, Atts, AttsSinceRevPos,
-    SendGzipAtts) ->
+    SendEncodedAtts) ->
     2 + % "--"
     size(Boundary) +
     36 + % "\r\ncontent-type: application/json\r\n\r\n"
@@ -388,7 +395,7 @@ len_doc_to_multi_part_stream(Boundary, JsonBytes, Atts, AttsSinceRevPos,
             if RevPos > AttsSinceRevPos ->
                 AccAttsSize +  
                 4 + % "\r\n\r\n"
-                case SendGzipAtts of
+                case SendEncodedAtts of
                 true ->
                     Att#att.att_len;
                 _ ->
@@ -403,31 +410,29 @@ len_doc_to_multi_part_stream(Boundary, JsonBytes, Atts, AttsSinceRevPos,
     2. % "--"
 
 doc_to_multi_part_stream(Boundary, JsonBytes, Atts, AttsSinceRevPos, WriteFun,
-    SendGzipAtts) ->
+    SendEncodedAtts) ->
     WriteFun([<<"--", Boundary/binary,
             "\r\ncontent-type: application/json\r\n\r\n">>,
             JsonBytes, <<"\r\n--", Boundary/binary>>]),
-    atts_to_mp(Atts, Boundary, WriteFun, AttsSinceRevPos, SendGzipAtts).
+    atts_to_mp(Atts, Boundary, WriteFun, AttsSinceRevPos, SendEncodedAtts).
 
-atts_to_mp([], _Boundary, WriteFun, _AttsSinceRevPos, _SendGzipAtts) ->
+atts_to_mp([], _Boundary, WriteFun, _AttsSinceRevPos, _SendEncAtts) ->
     WriteFun(<<"--">>);
 atts_to_mp([#att{revpos=RevPos} = Att | RestAtts], Boundary, WriteFun, 
-        AttsSinceRevPos, SendGzipAtts) when RevPos > AttsSinceRevPos ->
+        AttsSinceRevPos, SendEncodedAtts) when RevPos > AttsSinceRevPos ->
     WriteFun(<<"\r\n\r\n">>),
-    AttFun = case {Att#att.comp, SendGzipAtts} of
-    {true, false} ->
-        fun att_foldl_unzip/3;
-    _ ->
-        % receiver knows that the attachment is compressed by checking that the
-        % "gzip_length" field is present in the corresponding JSON attachment
-        % object found within the JSON doc
+    AttFun = case SendEncodedAtts of
+    false ->
+        fun att_foldl_decode/3;
+    true ->
         fun att_foldl/3
     end,
     AttFun(Att, fun(Data, ok) -> WriteFun(Data) end, ok),
     WriteFun(<<"\r\n--", Boundary/binary>>),
-    atts_to_mp(RestAtts, Boundary, WriteFun, AttsSinceRevPos, SendGzipAtts);
-atts_to_mp([_ | RestAtts], Boundary, WriteFun, AttsSinceRevPos, SendGzipAtts) ->
-    atts_to_mp(RestAtts, Boundary, WriteFun, AttsSinceRevPos, SendGzipAtts).
+    atts_to_mp(RestAtts, Boundary, WriteFun, AttsSinceRevPos, SendEncodedAtts);
+atts_to_mp([_ | RestAtts], Boundary, WriteFun, AttsSinceRevPos,
+    SendEncodedAtts) ->
+    atts_to_mp(RestAtts, Boundary, WriteFun, AttsSinceRevPos, SendEncodedAtts).
 
 
 doc_from_multi_part_stream(ContentType, DataFun) ->