summaryrefslogtreecommitdiff
path: root/src/mochiweb/mochiutf8.erl
diff options
context:
space:
mode:
Diffstat (limited to 'src/mochiweb/mochiutf8.erl')
-rw-r--r--src/mochiweb/mochiutf8.erl316
1 files changed, 0 insertions, 316 deletions
diff --git a/src/mochiweb/mochiutf8.erl b/src/mochiweb/mochiutf8.erl
deleted file mode 100644
index 206e1186..00000000
--- a/src/mochiweb/mochiutf8.erl
+++ /dev/null
@@ -1,316 +0,0 @@
-%% @copyright 2010 Mochi Media, Inc.
-%% @author Bob Ippolito <bob@mochimedia.com>
-
-%% @doc Algorithm to convert any binary to a valid UTF-8 sequence by ignoring
-%% invalid bytes.
-
--module(mochiutf8).
--export([valid_utf8_bytes/1, codepoint_to_bytes/1, bytes_to_codepoints/1]).
--export([bytes_foldl/3, codepoint_foldl/3, read_codepoint/1, len/1]).
-
-%% External API
-
--type unichar_low() :: 0..16#d7ff.
--type unichar_high() :: 16#e000..16#10ffff.
--type unichar() :: unichar_low() | unichar_high().
-
--spec codepoint_to_bytes(unichar()) -> binary().
-%% @doc Convert a unicode codepoint to UTF-8 bytes.
-codepoint_to_bytes(C) when (C >= 16#00 andalso C =< 16#7f) ->
- %% U+0000 - U+007F - 7 bits
- <<C>>;
-codepoint_to_bytes(C) when (C >= 16#080 andalso C =< 16#07FF) ->
- %% U+0080 - U+07FF - 11 bits
- <<0:5, B1:5, B0:6>> = <<C:16>>,
- <<2#110:3, B1:5,
- 2#10:2, B0:6>>;
-codepoint_to_bytes(C) when (C >= 16#0800 andalso C =< 16#FFFF) andalso
- (C < 16#D800 orelse C > 16#DFFF) ->
- %% U+0800 - U+FFFF - 16 bits (excluding UTC-16 surrogate code points)
- <<B2:4, B1:6, B0:6>> = <<C:16>>,
- <<2#1110:4, B2:4,
- 2#10:2, B1:6,
- 2#10:2, B0:6>>;
-codepoint_to_bytes(C) when (C >= 16#010000 andalso C =< 16#10FFFF) ->
- %% U+10000 - U+10FFFF - 21 bits
- <<0:3, B3:3, B2:6, B1:6, B0:6>> = <<C:24>>,
- <<2#11110:5, B3:3,
- 2#10:2, B2:6,
- 2#10:2, B1:6,
- 2#10:2, B0:6>>.
-
--spec codepoints_to_bytes([unichar()]) -> binary().
-%% @doc Convert a list of codepoints to a UTF-8 binary.
-codepoints_to_bytes(L) ->
- <<<<(codepoint_to_bytes(C))/binary>> || C <- L>>.
-
--spec read_codepoint(binary()) -> {unichar(), binary(), binary()}.
-read_codepoint(Bin = <<2#0:1, C:7, Rest/binary>>) ->
- %% U+0000 - U+007F - 7 bits
- <<B:1/binary, _/binary>> = Bin,
- {C, B, Rest};
-read_codepoint(Bin = <<2#110:3, B1:5,
- 2#10:2, B0:6,
- Rest/binary>>) ->
- %% U+0080 - U+07FF - 11 bits
- case <<B1:5, B0:6>> of
- <<C:11>> when C >= 16#80 ->
- <<B:2/binary, _/binary>> = Bin,
- {C, B, Rest}
- end;
-read_codepoint(Bin = <<2#1110:4, B2:4,
- 2#10:2, B1:6,
- 2#10:2, B0:6,
- Rest/binary>>) ->
- %% U+0800 - U+FFFF - 16 bits (excluding UTC-16 surrogate code points)
- case <<B2:4, B1:6, B0:6>> of
- <<C:16>> when (C >= 16#0800 andalso C =< 16#FFFF) andalso
- (C < 16#D800 orelse C > 16#DFFF) ->
- <<B:3/binary, _/binary>> = Bin,
- {C, B, Rest}
- end;
-read_codepoint(Bin = <<2#11110:5, B3:3,
- 2#10:2, B2:6,
- 2#10:2, B1:6,
- 2#10:2, B0:6,
- Rest/binary>>) ->
- %% U+10000 - U+10FFFF - 21 bits
- case <<B3:3, B2:6, B1:6, B0:6>> of
- <<C:21>> when (C >= 16#010000 andalso C =< 16#10FFFF) ->
- <<B:4/binary, _/binary>> = Bin,
- {C, B, Rest}
- end.
-
--spec codepoint_foldl(fun((unichar(), _) -> _), _, binary()) -> _.
-codepoint_foldl(F, Acc, <<>>) when is_function(F, 2) ->
- Acc;
-codepoint_foldl(F, Acc, Bin) ->
- {C, _, Rest} = read_codepoint(Bin),
- codepoint_foldl(F, F(C, Acc), Rest).
-
--spec bytes_foldl(fun((binary(), _) -> _), _, binary()) -> _.
-bytes_foldl(F, Acc, <<>>) when is_function(F, 2) ->
- Acc;
-bytes_foldl(F, Acc, Bin) ->
- {_, B, Rest} = read_codepoint(Bin),
- bytes_foldl(F, F(B, Acc), Rest).
-
--spec bytes_to_codepoints(binary()) -> [unichar()].
-bytes_to_codepoints(B) ->
- lists:reverse(codepoint_foldl(fun (C, Acc) -> [C | Acc] end, [], B)).
-
--spec len(binary()) -> non_neg_integer().
-len(<<>>) ->
- 0;
-len(B) ->
- {_, _, Rest} = read_codepoint(B),
- 1 + len(Rest).
-
--spec valid_utf8_bytes(B::binary()) -> binary().
-%% @doc Return only the bytes in B that represent valid UTF-8. Uses
-%% the following recursive algorithm: skip one byte if B does not
-%% follow UTF-8 syntax (a 1-4 byte encoding of some number),
-%% skip sequence of 2-4 bytes if it represents an overlong encoding
-%% or bad code point (surrogate U+D800 - U+DFFF or > U+10FFFF).
-valid_utf8_bytes(B) when is_binary(B) ->
- binary_skip_bytes(B, invalid_utf8_indexes(B)).
-
-%% Internal API
-
--spec binary_skip_bytes(binary(), [non_neg_integer()]) -> binary().
-%% @doc Return B, but skipping the 0-based indexes in L.
-binary_skip_bytes(B, []) ->
- B;
-binary_skip_bytes(B, L) ->
- binary_skip_bytes(B, L, 0, []).
-
-%% @private
--spec binary_skip_bytes(binary(), [non_neg_integer()], non_neg_integer(), iolist()) -> binary().
-binary_skip_bytes(B, [], _N, Acc) ->
- iolist_to_binary(lists:reverse([B | Acc]));
-binary_skip_bytes(<<_, RestB/binary>>, [N | RestL], N, Acc) ->
- binary_skip_bytes(RestB, RestL, 1 + N, Acc);
-binary_skip_bytes(<<C, RestB/binary>>, L, N, Acc) ->
- binary_skip_bytes(RestB, L, 1 + N, [C | Acc]).
-
--spec invalid_utf8_indexes(binary()) -> [non_neg_integer()].
-%% @doc Return the 0-based indexes in B that are not valid UTF-8.
-invalid_utf8_indexes(B) ->
- invalid_utf8_indexes(B, 0, []).
-
-%% @private.
--spec invalid_utf8_indexes(binary(), non_neg_integer(), [non_neg_integer()]) -> [non_neg_integer()].
-invalid_utf8_indexes(<<C, Rest/binary>>, N, Acc) when C < 16#80 ->
- %% U+0000 - U+007F - 7 bits
- invalid_utf8_indexes(Rest, 1 + N, Acc);
-invalid_utf8_indexes(<<C1, C2, Rest/binary>>, N, Acc)
- when C1 band 16#E0 =:= 16#C0,
- C2 band 16#C0 =:= 16#80 ->
- %% U+0080 - U+07FF - 11 bits
- case ((C1 band 16#1F) bsl 6) bor (C2 band 16#3F) of
- C when C < 16#80 ->
- %% Overlong encoding.
- invalid_utf8_indexes(Rest, 2 + N, [1 + N, N | Acc]);
- _ ->
- %% Upper bound U+07FF does not need to be checked
- invalid_utf8_indexes(Rest, 2 + N, Acc)
- end;
-invalid_utf8_indexes(<<C1, C2, C3, Rest/binary>>, N, Acc)
- when C1 band 16#F0 =:= 16#E0,
- C2 band 16#C0 =:= 16#80,
- C3 band 16#C0 =:= 16#80 ->
- %% U+0800 - U+FFFF - 16 bits
- case ((((C1 band 16#0F) bsl 6) bor (C2 band 16#3F)) bsl 6) bor
- (C3 band 16#3F) of
- C when (C < 16#800) orelse (C >= 16#D800 andalso C =< 16#DFFF) ->
- %% Overlong encoding or surrogate.
- invalid_utf8_indexes(Rest, 3 + N, [2 + N, 1 + N, N | Acc]);
- _ ->
- %% Upper bound U+FFFF does not need to be checked
- invalid_utf8_indexes(Rest, 3 + N, Acc)
- end;
-invalid_utf8_indexes(<<C1, C2, C3, C4, Rest/binary>>, N, Acc)
- when C1 band 16#F8 =:= 16#F0,
- C2 band 16#C0 =:= 16#80,
- C3 band 16#C0 =:= 16#80,
- C4 band 16#C0 =:= 16#80 ->
- %% U+10000 - U+10FFFF - 21 bits
- case ((((((C1 band 16#0F) bsl 6) bor (C2 band 16#3F)) bsl 6) bor
- (C3 band 16#3F)) bsl 6) bor (C4 band 16#3F) of
- C when (C < 16#10000) orelse (C > 16#10FFFF) ->
- %% Overlong encoding or invalid code point.
- invalid_utf8_indexes(Rest, 4 + N, [3 + N, 2 + N, 1 + N, N | Acc]);
- _ ->
- invalid_utf8_indexes(Rest, 4 + N, Acc)
- end;
-invalid_utf8_indexes(<<_, Rest/binary>>, N, Acc) ->
- %% Invalid char
- invalid_utf8_indexes(Rest, 1 + N, [N | Acc]);
-invalid_utf8_indexes(<<>>, _N, Acc) ->
- lists:reverse(Acc).
-
-%%
-%% Tests
-%%
--include_lib("eunit/include/eunit.hrl").
--ifdef(TEST).
-
-binary_skip_bytes_test() ->
- ?assertEqual(<<"foo">>,
- binary_skip_bytes(<<"foo">>, [])),
- ?assertEqual(<<"foobar">>,
- binary_skip_bytes(<<"foo bar">>, [3])),
- ?assertEqual(<<"foo">>,
- binary_skip_bytes(<<"foo bar">>, [3, 4, 5, 6])),
- ?assertEqual(<<"oo bar">>,
- binary_skip_bytes(<<"foo bar">>, [0])),
- ok.
-
-invalid_utf8_indexes_test() ->
- ?assertEqual(
- [],
- invalid_utf8_indexes(<<"unicode snowman for you: ", 226, 152, 131>>)),
- ?assertEqual(
- [0],
- invalid_utf8_indexes(<<128>>)),
- ?assertEqual(
- [57,59,60,64,66,67],
- invalid_utf8_indexes(<<"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; (",
- 167, 65, 170, 186, 73, 83, 80, 166, 87, 186, 217, 41, 41>>)),
- ok.
-
-codepoint_to_bytes_test() ->
- %% U+0000 - U+007F - 7 bits
- %% U+0080 - U+07FF - 11 bits
- %% U+0800 - U+FFFF - 16 bits (excluding UTC-16 surrogate code points)
- %% U+10000 - U+10FFFF - 21 bits
- ?assertEqual(
- <<"a">>,
- codepoint_to_bytes($a)),
- ?assertEqual(
- <<16#c2, 16#80>>,
- codepoint_to_bytes(16#80)),
- ?assertEqual(
- <<16#df, 16#bf>>,
- codepoint_to_bytes(16#07ff)),
- ?assertEqual(
- <<16#ef, 16#bf, 16#bf>>,
- codepoint_to_bytes(16#ffff)),
- ?assertEqual(
- <<16#f4, 16#8f, 16#bf, 16#bf>>,
- codepoint_to_bytes(16#10ffff)),
- ok.
-
-bytes_foldl_test() ->
- ?assertEqual(
- <<"abc">>,
- bytes_foldl(fun (B, Acc) -> <<Acc/binary, B/binary>> end, <<>>, <<"abc">>)),
- ?assertEqual(
- <<"abc", 226, 152, 131, 228, 184, 173, 194, 133, 244,143,191,191>>,
- bytes_foldl(fun (B, Acc) -> <<Acc/binary, B/binary>> end, <<>>,
- <<"abc", 226, 152, 131, 228, 184, 173, 194, 133, 244,143,191,191>>)),
- ok.
-
-bytes_to_codepoints_test() ->
- ?assertEqual(
- "abc" ++ [16#2603, 16#4e2d, 16#85, 16#10ffff],
- bytes_to_codepoints(<<"abc", 226, 152, 131, 228, 184, 173, 194, 133, 244,143,191,191>>)),
- ok.
-
-codepoint_foldl_test() ->
- ?assertEqual(
- "cba",
- codepoint_foldl(fun (C, Acc) -> [C | Acc] end, [], <<"abc">>)),
- ?assertEqual(
- [16#10ffff, 16#85, 16#4e2d, 16#2603 | "cba"],
- codepoint_foldl(fun (C, Acc) -> [C | Acc] end, [],
- <<"abc", 226, 152, 131, 228, 184, 173, 194, 133, 244,143,191,191>>)),
- ok.
-
-len_test() ->
- ?assertEqual(
- 29,
- len(<<"unicode snowman for you: ", 226, 152, 131, 228, 184, 173, 194, 133, 244, 143, 191, 191>>)),
- ok.
-
-codepoints_to_bytes_test() ->
- ?assertEqual(
- iolist_to_binary(lists:map(fun codepoint_to_bytes/1, lists:seq(1, 1000))),
- codepoints_to_bytes(lists:seq(1, 1000))),
- ok.
-
-valid_utf8_bytes_test() ->
- ?assertEqual(
- <<"invalid U+11ffff: ">>,
- valid_utf8_bytes(<<"invalid U+11ffff: ", 244, 159, 191, 191>>)),
- ?assertEqual(
- <<"U+10ffff: ", 244, 143, 191, 191>>,
- valid_utf8_bytes(<<"U+10ffff: ", 244, 143, 191, 191>>)),
- ?assertEqual(
- <<"overlong 2-byte encoding (a): ">>,
- valid_utf8_bytes(<<"overlong 2-byte encoding (a): ", 2#11000001, 2#10100001>>)),
- ?assertEqual(
- <<"overlong 2-byte encoding (!): ">>,
- valid_utf8_bytes(<<"overlong 2-byte encoding (!): ", 2#11000000, 2#10100001>>)),
- ?assertEqual(
- <<"mu: ", 194, 181>>,
- valid_utf8_bytes(<<"mu: ", 194, 181>>)),
- ?assertEqual(
- <<"bad coding bytes: ">>,
- valid_utf8_bytes(<<"bad coding bytes: ", 2#10011111, 2#10111111, 2#11111111>>)),
- ?assertEqual(
- <<"low surrogate (unpaired): ">>,
- valid_utf8_bytes(<<"low surrogate (unpaired): ", 237, 176, 128>>)),
- ?assertEqual(
- <<"high surrogate (unpaired): ">>,
- valid_utf8_bytes(<<"high surrogate (unpaired): ", 237, 191, 191>>)),
- ?assertEqual(
- <<"unicode snowman for you: ", 226, 152, 131>>,
- valid_utf8_bytes(<<"unicode snowman for you: ", 226, 152, 131>>)),
- ?assertEqual(
- <<"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; (AISPW))">>,
- valid_utf8_bytes(<<"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; (",
- 167, 65, 170, 186, 73, 83, 80, 166, 87, 186, 217, 41, 41>>)),
- ok.
-
--endif.
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-01 06:13:52 +0000