diff options
Diffstat (limited to '1.1.x/src/couchdb/couch_file.erl')
-rw-r--r-- | 1.1.x/src/couchdb/couch_file.erl | 614 |
1 files changed, 0 insertions, 614 deletions
diff --git a/1.1.x/src/couchdb/couch_file.erl b/1.1.x/src/couchdb/couch_file.erl deleted file mode 100644 index 7b677034..00000000 --- a/1.1.x/src/couchdb/couch_file.erl +++ /dev/null @@ -1,614 +0,0 @@ -% Licensed under the Apache License, Version 2.0 (the "License"); you may not -% use this file except in compliance with the License. You may obtain a copy of -% the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -% License for the specific language governing permissions and limitations under -% the License. - --module(couch_file). --behaviour(gen_server). - --include("couch_db.hrl"). - --define(SIZE_BLOCK, 4096). - --record(file, { - fd, - tail_append_begin = 0, % 09 UPGRADE CODE - eof = 0 - }). - --export([open/1, open/2, close/1, bytes/1, sync/1, append_binary/2,old_pread/3]). --export([append_term/2, pread_term/2, pread_iolist/2, write_header/2]). --export([pread_binary/2, read_header/1, truncate/2, upgrade_old_header/2]). --export([append_term_md5/2,append_binary_md5/2]). --export([init/1, terminate/2, handle_call/3, handle_cast/2, code_change/3, handle_info/2]). --export([delete/2,delete/3,init_delete_dir/1]). - -%%---------------------------------------------------------------------- -%% Args: Valid Options are [create] and [create,overwrite]. -%% Files are opened in read/write mode. -%% Returns: On success, {ok, Fd} -%% or {error, Reason} if the file could not be opened. -%%---------------------------------------------------------------------- - -open(Filepath) -> - open(Filepath, []). - -open(Filepath, Options) -> - case gen_server:start_link(couch_file, - {Filepath, Options, self(), Ref = make_ref()}, []) of - {ok, Fd} -> - {ok, Fd}; - ignore -> - % get the error - receive - {Ref, Pid, Error} -> - case process_info(self(), trap_exit) of - {trap_exit, true} -> receive {'EXIT', Pid, _} -> ok end; - {trap_exit, false} -> ok - end, - case Error of - {error, eacces} -> {file_permission_error, Filepath}; - _ -> Error - end - end; - Error -> - Error - end. - - -%%---------------------------------------------------------------------- -%% Purpose: To append an Erlang term to the end of the file. -%% Args: Erlang term to serialize and append to the file. -%% Returns: {ok, Pos} where Pos is the file offset to the beginning the -%% serialized term. Use pread_term to read the term back. -%% or {error, Reason}. -%%---------------------------------------------------------------------- - -append_term(Fd, Term) -> - append_binary(Fd, term_to_binary(Term)). - -append_term_md5(Fd, Term) -> - append_binary_md5(Fd, term_to_binary(Term)). - - -%%---------------------------------------------------------------------- -%% Purpose: To append an Erlang binary to the end of the file. -%% Args: Erlang term to serialize and append to the file. -%% Returns: {ok, Pos} where Pos is the file offset to the beginning the -%% serialized term. Use pread_term to read the term back. -%% or {error, Reason}. -%%---------------------------------------------------------------------- - -append_binary(Fd, Bin) -> - Size = iolist_size(Bin), - gen_server:call(Fd, {append_bin, - [<<0:1/integer,Size:31/integer>>, Bin]}, infinity). - -append_binary_md5(Fd, Bin) -> - Size = iolist_size(Bin), - gen_server:call(Fd, {append_bin, - [<<1:1/integer,Size:31/integer>>, couch_util:md5(Bin), Bin]}, infinity). - - -%%---------------------------------------------------------------------- -%% Purpose: Reads a term from a file that was written with append_term -%% Args: Pos, the offset into the file where the term is serialized. -%% Returns: {ok, Term} -%% or {error, Reason}. -%%---------------------------------------------------------------------- - - -pread_term(Fd, Pos) -> - {ok, Bin} = pread_binary(Fd, Pos), - {ok, binary_to_term(Bin)}. - - -%%---------------------------------------------------------------------- -%% Purpose: Reads a binrary from a file that was written with append_binary -%% Args: Pos, the offset into the file where the term is serialized. -%% Returns: {ok, Term} -%% or {error, Reason}. -%%---------------------------------------------------------------------- - -pread_binary(Fd, Pos) -> - {ok, L} = pread_iolist(Fd, Pos), - {ok, iolist_to_binary(L)}. - - -pread_iolist(Fd, Pos) -> - case gen_server:call(Fd, {pread_iolist, Pos}, infinity) of - {ok, IoList, <<>>} -> - {ok, IoList}; - {ok, IoList, Md5} -> - case couch_util:md5(IoList) of - Md5 -> - {ok, IoList}; - _ -> - exit({file_corruption, <<"file corruption">>}) - end; - Error -> - Error - end. - -%%---------------------------------------------------------------------- -%% Purpose: The length of a file, in bytes. -%% Returns: {ok, Bytes} -%% or {error, Reason}. -%%---------------------------------------------------------------------- - -% length in bytes -bytes(Fd) -> - gen_server:call(Fd, bytes, infinity). - -%%---------------------------------------------------------------------- -%% Purpose: Truncate a file to the number of bytes. -%% Returns: ok -%% or {error, Reason}. -%%---------------------------------------------------------------------- - -truncate(Fd, Pos) -> - gen_server:call(Fd, {truncate, Pos}, infinity). - -%%---------------------------------------------------------------------- -%% Purpose: Ensure all bytes written to the file are flushed to disk. -%% Returns: ok -%% or {error, Reason}. -%%---------------------------------------------------------------------- - -sync(Filepath) when is_list(Filepath) -> - {ok, Fd} = file:open(Filepath, [append, raw]), - try file:sync(Fd) after file:close(Fd) end; -sync(Fd) -> - gen_server:call(Fd, sync, infinity). - -%%---------------------------------------------------------------------- -%% Purpose: Close the file. -%% Returns: ok -%%---------------------------------------------------------------------- -close(Fd) -> - couch_util:shutdown_sync(Fd). - - -delete(RootDir, Filepath) -> - delete(RootDir, Filepath, true). - - -delete(RootDir, Filepath, Async) -> - DelFile = filename:join([RootDir,".delete", ?b2l(couch_uuids:random())]), - case file:rename(Filepath, DelFile) of - ok -> - if (Async) -> - spawn(file, delete, [DelFile]), - ok; - true -> - file:delete(DelFile) - end; - Error -> - Error - end. - - -init_delete_dir(RootDir) -> - Dir = filename:join(RootDir,".delete"), - % note: ensure_dir requires an actual filename companent, which is the - % reason for "foo". - filelib:ensure_dir(filename:join(Dir,"foo")), - filelib:fold_files(Dir, ".*", true, - fun(Filename, _) -> - ok = file:delete(Filename) - end, ok). - - -% 09 UPGRADE CODE -old_pread(Fd, Pos, Len) -> - {ok, <<RawBin:Len/binary>>, false} = gen_server:call(Fd, {pread, Pos, Len}, infinity), - {ok, RawBin}. - -% 09 UPGRADE CODE -upgrade_old_header(Fd, Sig) -> - gen_server:call(Fd, {upgrade_old_header, Sig}, infinity). - - -read_header(Fd) -> - case gen_server:call(Fd, find_header, infinity) of - {ok, Bin} -> - {ok, binary_to_term(Bin)}; - Else -> - Else - end. - -write_header(Fd, Data) -> - Bin = term_to_binary(Data), - Md5 = couch_util:md5(Bin), - % now we assemble the final header binary and write to disk - FinalBin = <<Md5/binary, Bin/binary>>, - gen_server:call(Fd, {write_header, FinalBin}, infinity). - - - - -init_status_error(ReturnPid, Ref, Error) -> - ReturnPid ! {Ref, self(), Error}, - ignore. - -% server functions - -init({Filepath, Options, ReturnPid, Ref}) -> - process_flag(trap_exit, true), - case lists:member(create, Options) of - true -> - filelib:ensure_dir(Filepath), - case file:open(Filepath, [read, append, raw, binary]) of - {ok, Fd} -> - {ok, Length} = file:position(Fd, eof), - case Length > 0 of - true -> - % this means the file already exists and has data. - % FYI: We don't differentiate between empty files and non-existant - % files here. - case lists:member(overwrite, Options) of - true -> - {ok, 0} = file:position(Fd, 0), - ok = file:truncate(Fd), - ok = file:sync(Fd), - maybe_track_open_os_files(Options), - {ok, #file{fd=Fd}}; - false -> - ok = file:close(Fd), - init_status_error(ReturnPid, Ref, file_exists) - end; - false -> - maybe_track_open_os_files(Options), - {ok, #file{fd=Fd}} - end; - Error -> - init_status_error(ReturnPid, Ref, Error) - end; - false -> - % open in read mode first, so we don't create the file if it doesn't exist. - case file:open(Filepath, [read, raw]) of - {ok, Fd_Read} -> - {ok, Fd} = file:open(Filepath, [read, append, raw, binary]), - ok = file:close(Fd_Read), - maybe_track_open_os_files(Options), - {ok, Length} = file:position(Fd, eof), - {ok, #file{fd=Fd, eof=Length}}; - Error -> - init_status_error(ReturnPid, Ref, Error) - end - end. - -maybe_track_open_os_files(FileOptions) -> - case lists:member(sys_db, FileOptions) of - true -> - ok; - false -> - couch_stats_collector:track_process_count({couchdb, open_os_files}) - end. - -terminate(_Reason, #file{fd = Fd}) -> - ok = file:close(Fd). - - -handle_call({pread_iolist, Pos}, _From, File) -> - {RawData, NextPos} = try - % up to 8Kbs of read ahead - read_raw_iolist_int(File, Pos, 2 * ?SIZE_BLOCK - (Pos rem ?SIZE_BLOCK)) - catch - _:_ -> - read_raw_iolist_int(File, Pos, 4) - end, - <<Prefix:1/integer, Len:31/integer, RestRawData/binary>> = - iolist_to_binary(RawData), - case Prefix of - 1 -> - {Md5, IoList} = extract_md5( - maybe_read_more_iolist(RestRawData, 16 + Len, NextPos, File)), - {reply, {ok, IoList, Md5}, File}; - 0 -> - IoList = maybe_read_more_iolist(RestRawData, Len, NextPos, File), - {reply, {ok, IoList, <<>>}, File} - end; -handle_call({pread, Pos, Bytes}, _From, #file{fd=Fd,tail_append_begin=TailAppendBegin}=File) -> - {ok, Bin} = file:pread(Fd, Pos, Bytes), - {reply, {ok, Bin, Pos >= TailAppendBegin}, File}; -handle_call(bytes, _From, #file{eof=Length}=File) -> - {reply, {ok, Length}, File}; -handle_call(sync, _From, #file{fd=Fd}=File) -> - {reply, file:sync(Fd), File}; -handle_call({truncate, Pos}, _From, #file{fd=Fd}=File) -> - {ok, Pos} = file:position(Fd, Pos), - case file:truncate(Fd) of - ok -> - {reply, ok, File#file{eof=Pos}}; - Error -> - {reply, Error, File} - end; -handle_call({append_bin, Bin}, _From, #file{fd=Fd, eof=Pos}=File) -> - Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin), - case file:write(Fd, Blocks) of - ok -> - {reply, {ok, Pos}, File#file{eof=Pos+iolist_size(Blocks)}}; - Error -> - {reply, Error, File} - end; -handle_call({write_header, Bin}, _From, #file{fd=Fd, eof=Pos}=File) -> - BinSize = size(Bin), - case Pos rem ?SIZE_BLOCK of - 0 -> - Padding = <<>>; - BlockOffset -> - Padding = <<0:(8*(?SIZE_BLOCK-BlockOffset))>> - end, - FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])], - case file:write(Fd, FinalBin) of - ok -> - {reply, ok, File#file{eof=Pos+iolist_size(FinalBin)}}; - Error -> - {reply, Error, File} - end; - - -handle_call({upgrade_old_header, Prefix}, _From, #file{fd=Fd}=File) -> - case (catch read_old_header(Fd, Prefix)) of - {ok, Header} -> - TailAppendBegin = File#file.eof, - Bin = term_to_binary(Header), - Md5 = couch_util:md5(Bin), - % now we assemble the final header binary and write to disk - FinalBin = <<Md5/binary, Bin/binary>>, - {reply, ok, _} = handle_call({write_header, FinalBin}, ok, File), - ok = write_old_header(Fd, <<"upgraded">>, TailAppendBegin), - {reply, ok, File#file{tail_append_begin=TailAppendBegin}}; - _Error -> - case (catch read_old_header(Fd, <<"upgraded">>)) of - {ok, TailAppendBegin} -> - {reply, ok, File#file{tail_append_begin = TailAppendBegin}}; - _Error2 -> - {reply, ok, File} - end - end; - - -handle_call(find_header, _From, #file{fd=Fd, eof=Pos}=File) -> - {reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}. - -% 09 UPGRADE CODE --define(HEADER_SIZE, 2048). % size of each segment of the doubly written header - -% 09 UPGRADE CODE -read_old_header(Fd, Prefix) -> - {ok, Bin} = file:pread(Fd, 0, 2*(?HEADER_SIZE)), - <<Bin1:(?HEADER_SIZE)/binary, Bin2:(?HEADER_SIZE)/binary>> = Bin, - Result = - % read the first header - case extract_header(Prefix, Bin1) of - {ok, Header1} -> - case extract_header(Prefix, Bin2) of - {ok, Header2} -> - case Header1 == Header2 of - true -> - % Everything is completely normal! - {ok, Header1}; - false -> - % To get here we must have two different header versions with signatures intact. - % It's weird but possible (a commit failure right at the 2k boundary). Log it and take the first. - ?LOG_INFO("Header version differences.~nPrimary Header: ~p~nSecondary Header: ~p", [Header1, Header2]), - {ok, Header1} - end; - Error -> - % error reading second header. It's ok, but log it. - ?LOG_INFO("Secondary header corruption (error: ~p). Using primary header.", [Error]), - {ok, Header1} - end; - Error -> - % error reading primary header - case extract_header(Prefix, Bin2) of - {ok, Header2} -> - % log corrupt primary header. It's ok since the secondary is still good. - ?LOG_INFO("Primary header corruption (error: ~p). Using secondary header.", [Error]), - {ok, Header2}; - _ -> - % error reading secondary header too - % return the error, no need to log anything as the caller will be responsible for dealing with the error. - Error - end - end, - case Result of - {ok, {pointer_to_header_data, Ptr}} -> - pread_term(Fd, Ptr); - _ -> - Result - end. - -% 09 UPGRADE CODE -extract_header(Prefix, Bin) -> - SizeOfPrefix = size(Prefix), - SizeOfTermBin = ?HEADER_SIZE - - SizeOfPrefix - - 16, % md5 sig - - <<HeaderPrefix:SizeOfPrefix/binary, TermBin:SizeOfTermBin/binary, Sig:16/binary>> = Bin, - - % check the header prefix - case HeaderPrefix of - Prefix -> - % check the integrity signature - case couch_util:md5(TermBin) == Sig of - true -> - Header = binary_to_term(TermBin), - {ok, Header}; - false -> - header_corrupt - end; - _ -> - unknown_header_type - end. - - -% 09 UPGRADE CODE -write_old_header(Fd, Prefix, Data) -> - TermBin = term_to_binary(Data), - % the size of all the bytes written to the header, including the md5 signature (16 bytes) - FilledSize = byte_size(Prefix) + byte_size(TermBin) + 16, - {TermBin2, FilledSize2} = - case FilledSize > ?HEADER_SIZE of - true -> - % too big! - {ok, Pos} = append_binary(Fd, TermBin), - PtrBin = term_to_binary({pointer_to_header_data, Pos}), - {PtrBin, byte_size(Prefix) + byte_size(PtrBin) + 16}; - false -> - {TermBin, FilledSize} - end, - ok = file:sync(Fd), - % pad out the header with zeros, then take the md5 hash - PadZeros = <<0:(8*(?HEADER_SIZE - FilledSize2))>>, - Sig = couch_util:md5([TermBin2, PadZeros]), - % now we assemble the final header binary and write to disk - WriteBin = <<Prefix/binary, TermBin2/binary, PadZeros/binary, Sig/binary>>, - ?HEADER_SIZE = size(WriteBin), % sanity check - DblWriteBin = [WriteBin, WriteBin], - ok = file:pwrite(Fd, 0, DblWriteBin), - ok = file:sync(Fd). - - -handle_cast(close, Fd) -> - {stop,normal,Fd}. - - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -handle_info({'EXIT', _, normal}, Fd) -> - {noreply, Fd}; -handle_info({'EXIT', _, Reason}, Fd) -> - {stop, Reason, Fd}. - - -find_header(_Fd, -1) -> - no_valid_header; -find_header(Fd, Block) -> - case (catch load_header(Fd, Block)) of - {ok, Bin} -> - {ok, Bin}; - _Error -> - find_header(Fd, Block -1) - end. - -load_header(Fd, Block) -> - {ok, <<1, HeaderLen:32/integer, RestBlock/binary>>} = - file:pread(Fd, Block * ?SIZE_BLOCK, ?SIZE_BLOCK), - TotalBytes = calculate_total_read_len(1, HeaderLen), - case TotalBytes > byte_size(RestBlock) of - false -> - <<RawBin:TotalBytes/binary, _/binary>> = RestBlock; - true -> - {ok, Missing} = file:pread( - Fd, (Block * ?SIZE_BLOCK) + 5 + byte_size(RestBlock), - TotalBytes - byte_size(RestBlock)), - RawBin = <<RestBlock/binary, Missing/binary>> - end, - <<Md5Sig:16/binary, HeaderBin/binary>> = - iolist_to_binary(remove_block_prefixes(1, RawBin)), - Md5Sig = couch_util:md5(HeaderBin), - {ok, HeaderBin}. - -maybe_read_more_iolist(Buffer, DataSize, _, _) - when DataSize =< byte_size(Buffer) -> - <<Data:DataSize/binary, _/binary>> = Buffer, - [Data]; -maybe_read_more_iolist(Buffer, DataSize, NextPos, File) -> - {Missing, _} = - read_raw_iolist_int(File, NextPos, DataSize - byte_size(Buffer)), - [Buffer, Missing]. - --spec read_raw_iolist_int(#file{}, Pos::non_neg_integer(), Len::non_neg_integer()) -> - {Data::iolist(), CurPos::non_neg_integer()}. -read_raw_iolist_int(Fd, {Pos, _Size}, Len) -> % 0110 UPGRADE CODE - read_raw_iolist_int(Fd, Pos, Len); -read_raw_iolist_int(#file{fd=Fd, tail_append_begin=TAB}, Pos, Len) -> - BlockOffset = Pos rem ?SIZE_BLOCK, - TotalBytes = calculate_total_read_len(BlockOffset, Len), - {ok, <<RawBin:TotalBytes/binary>>} = file:pread(Fd, Pos, TotalBytes), - if Pos >= TAB -> - {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes}; - true -> - % 09 UPGRADE CODE - <<ReturnBin:Len/binary, _/binary>> = RawBin, - {[ReturnBin], Pos + Len} - end. - --spec extract_md5(iolist()) -> {binary(), iolist()}. -extract_md5(FullIoList) -> - {Md5List, IoList} = split_iolist(FullIoList, 16, []), - {iolist_to_binary(Md5List), IoList}. - -calculate_total_read_len(0, FinalLen) -> - calculate_total_read_len(1, FinalLen) + 1; -calculate_total_read_len(BlockOffset, FinalLen) -> - case ?SIZE_BLOCK - BlockOffset of - BlockLeft when BlockLeft >= FinalLen -> - FinalLen; - BlockLeft -> - FinalLen + ((FinalLen - BlockLeft) div (?SIZE_BLOCK -1)) + - if ((FinalLen - BlockLeft) rem (?SIZE_BLOCK -1)) =:= 0 -> 0; - true -> 1 end - end. - -remove_block_prefixes(_BlockOffset, <<>>) -> - []; -remove_block_prefixes(0, <<_BlockPrefix,Rest/binary>>) -> - remove_block_prefixes(1, Rest); -remove_block_prefixes(BlockOffset, Bin) -> - BlockBytesAvailable = ?SIZE_BLOCK - BlockOffset, - case size(Bin) of - Size when Size > BlockBytesAvailable -> - <<DataBlock:BlockBytesAvailable/binary,Rest/binary>> = Bin, - [DataBlock | remove_block_prefixes(0, Rest)]; - _Size -> - [Bin] - end. - -make_blocks(_BlockOffset, []) -> - []; -make_blocks(0, IoList) -> - [<<0>> | make_blocks(1, IoList)]; -make_blocks(BlockOffset, IoList) -> - case split_iolist(IoList, (?SIZE_BLOCK - BlockOffset), []) of - {Begin, End} -> - [Begin | make_blocks(0, End)]; - _SplitRemaining -> - IoList - end. - -%% @doc Returns a tuple where the first element contains the leading SplitAt -%% bytes of the original iolist, and the 2nd element is the tail. If SplitAt -%% is larger than byte_size(IoList), return the difference. --spec split_iolist(IoList::iolist(), SplitAt::non_neg_integer(), Acc::list()) -> - {iolist(), iolist()} | non_neg_integer(). -split_iolist(List, 0, BeginAcc) -> - {lists:reverse(BeginAcc), List}; -split_iolist([], SplitAt, _BeginAcc) -> - SplitAt; -split_iolist([<<Bin/binary>> | Rest], SplitAt, BeginAcc) when SplitAt > byte_size(Bin) -> - split_iolist(Rest, SplitAt - byte_size(Bin), [Bin | BeginAcc]); -split_iolist([<<Bin/binary>> | Rest], SplitAt, BeginAcc) -> - <<Begin:SplitAt/binary,End/binary>> = Bin, - split_iolist([End | Rest], 0, [Begin | BeginAcc]); -split_iolist([Sublist| Rest], SplitAt, BeginAcc) when is_list(Sublist) -> - case split_iolist(Sublist, SplitAt, BeginAcc) of - {Begin, End} -> - {Begin, [End | Rest]}; - SplitRemaining -> - split_iolist(Rest, SplitAt - (SplitAt - SplitRemaining), [Sublist | BeginAcc]) - end; -split_iolist([Byte | Rest], SplitAt, BeginAcc) when is_integer(Byte) -> - split_iolist(Rest, SplitAt - 1, [Byte | BeginAcc]). |