path: root/src/etap/etap.erl

%% Copyright (c) 2008-2009 Nick Gerakines <nick@gerakines.net>
%% 
%% Permission is hereby granted, free of charge, to any person
%% obtaining a copy of this software and associated documentation
%% files (the "Software"), to deal in the Software without
%% restriction, including without limitation the rights to use,
%% copy, modify, merge, publish, distribute, sublicense, and/or sell
%% copies of the Software, and to permit persons to whom the
%% Software is furnished to do so, subject to the following
%% conditions:
%% 
%% The above copyright notice and this permission notice shall be
%% included in all copies or substantial portions of the Software.
%% 
%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
%% OTHER DEALINGS IN THE SOFTWARE.
%% 
%% @author Nick Gerakines <nick@gerakines.net> [http://socklabs.com/]
%% @author Jeremy Wall <jeremy@marzhillstudios.com>
%% @version 0.3.4
%% @copyright 2007-2008 Jeremy Wall, 2008-2009 Nick Gerakines
%% @reference http://testanything.org/wiki/index.php/Main_Page
%% @reference http://en.wikipedia.org/wiki/Test_Anything_Protocol
%% @todo Finish implementing the skip directive.
%% @todo Document the messages handled by this receive loop.
%% @todo Explain in documentation why we use a process to handle test input.
%% @doc etap is a TAP testing module for Erlang components and applications.
%% This module allows developers to test their software using the TAP method.
%% 
%% <blockquote cite="http://en.wikipedia.org/wiki/Test_Anything_Protocol"><p>
%% TAP, the Test Anything Protocol, is a simple text-based interface between
%% testing modules in a test harness. TAP started life as part of the test
%% harness for Perl but now has implementations in C/C++, Python, PHP, Perl
%% and probably others by the time you read this.
%% </p></blockquote>
%% 
%% The testing process begins by defining a plan using etap:plan/1, running
%% a number of etap tests and then calling eta:end_tests/0. Please refer to
%% the Erlang modules in the t directory of this project for example tests.
-module(etap).
-export([
    ensure_test_server/0, start_etap_server/0, test_server/1,
    diag/1, diag/2, plan/1, end_tests/0, not_ok/2, ok/2, is/3, isnt/3,
    any/3, none/3, fun_is/3, is_greater/3, skip/1, skip/2,
    ensure_coverage_starts/0, ensure_coverage_ends/0, coverage_report/0,
    datetime/1, skip/3, bail/0, bail/1
]).
-record(test_state, {planned = 0, count = 0, pass = 0, fail = 0, skip = 0, skip_reason = ""}).
-vsn("0.3.4").

%% @spec plan(N) -> Result
%%       N = unknown | skip | {skip, string()} | integer()
%%       Result = ok
%% @doc Create a test plan and boot strap the test server.
plan(unknown) ->
    ensure_coverage_starts(),
    ensure_test_server(),
    etap_server ! {self(), plan, unknown},
    ok;
plan(skip) ->
    io:format("1..0 # skip~n");
plan({skip, Reason}) ->
    io:format("1..0 # skip ~s~n", [Reason]);
plan(N) when is_integer(N), N > 0 ->
    ensure_coverage_starts(),
    ensure_test_server(),
    etap_server ! {self(), plan, N},
    ok.

%% @spec end_tests() -> ok
%% @doc End the current test plan and output test results.
%% @todo This should probably be done in the test_server process.
end_tests() ->
    ensure_coverage_ends(),
    etap_server ! {self(), state},
    State = receive X -> X end,
    if
        State#test_state.planned == -1 ->
            io:format("1..~p~n", [State#test_state.count]);
        true ->
            ok
    end,
    case whereis(etap_server) of
        undefined -> ok;
        _ -> etap_server ! done, ok
    end.

%% @private
ensure_coverage_starts() ->
    case os:getenv("COVER") of
        false -> ok;
        _ ->
            BeamDir = case os:getenv("COVER_BIN") of false -> "ebin"; X -> X end,
            cover:compile_beam_directory(BeamDir)
    end.

%% @private
%% @doc Attempts to write out any collected coverage data to the cover/
%% directory. This function should not be called externally, but it could be.
ensure_coverage_ends() ->
    case os:getenv("COVER") of
        false -> ok;
        _ ->
            filelib:ensure_dir("cover/"),
            Name = lists:flatten([
                io_lib:format("~.16b", [X]) || X <- binary_to_list(erlang:md5(
                     term_to_binary({make_ref(), now()})
                ))
            ]),
            cover:export("cover/" ++ Name ++ ".coverdata")
    end.

%% @spec coverage_report() -> ok
%% @doc Use the cover module's covreage report builder to create code coverage
%% reports from recently created coverdata files.
coverage_report() ->
    [cover:import(File) || File <- filelib:wildcard("cover/*.coverdata")],
    lists:foreach(
        fun(Mod) ->
            cover:analyse_to_file(Mod, atom_to_list(Mod) ++ "_coverage.txt", [])
        end,
        cover:imported_modules()
    ),
    ok.

bail() ->
    bail("").

bail(Reason) ->
    etap_server ! {self(), diag, "Bail out! " ++ Reason},
    ensure_coverage_ends(),
    etap_server ! done, ok,
    ok.


%% @spec diag(S) -> ok
%%       S = string()
%% @doc Print a debug/status message related to the test suite.
diag(S) -> etap_server ! {self(), diag, "# " ++ S}, ok.

%% @spec diag(Format, Data) -> ok
%%      Format = atom() | string() | binary()
%%      Data = [term()]
%%      UnicodeList = [Unicode]
%%      Unicode = int()
%% @doc Print a debug/status message related to the test suite.
%% Function arguments are passed through io_lib:format/2.
diag(Format, Data) -> diag(io_lib:format(Format, Data)).

%% @spec ok(Expr, Desc) -> Result
%%       Expr = true | false
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that a statement is true.
ok(Expr, Desc) -> mk_tap(Expr == true, Desc).

%% @spec not_ok(Expr, Desc) -> Result
%%       Expr = true | false
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that a statement is false.
not_ok(Expr, Desc) -> mk_tap(Expr == false, Desc).

%% @spec is(Got, Expected, Desc) -> Result
%%       Got = any()
%%       Expected = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that two values are the same.
is(Got, Expected, Desc) ->
    case mk_tap(Got == Expected, Desc) of
        false ->
            etap_server ! {self(), diag, "    ---"},
            etap_server ! {self(), diag, io_lib:format("    description: ~p", [Desc])},
            etap_server ! {self(), diag, io_lib:format("    found:       ~p", [Got])},
            etap_server ! {self(), diag, io_lib:format("    wanted:      ~p", [Expected])},
            etap_server ! {self(), diag, "    ..."},
            false;
        true -> true
    end.

%% @spec isnt(Got, Expected, Desc) -> Result
%%       Got = any()
%%       Expected = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that two values are not the same.
isnt(Got, Expected, Desc) -> mk_tap(Got /= Expected, Desc).

%% @spec is_greater(ValueA, ValueB, Desc) -> Result
%%       ValueA = number()
%%       ValueB = number()
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that an integer is greater than another.
is_greater(ValueA, ValueB, Desc) when is_integer(ValueA), is_integer(ValueB) ->
    mk_tap(ValueA > ValueB, Desc).

%% @spec any(Got, Items, Desc) -> Result
%%       Got = any()
%%       Items = [any()]
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that an item is in a list.
any(Got, Items, Desc) ->
    is(lists:member(Got, Items), true, Desc).

%% @spec none(Got, Items, Desc) -> Result
%%       Got = any()
%%       Items = [any()]
%%       Desc = string()
%%       Result = true | false
%% @doc Assert that an item is not in a list.
none(Got, Items, Desc) ->
    is(lists:member(Got, Items), false, Desc).

%% @spec fun_is(Fun, Expected, Desc) -> Result
%%       Fun = function()
%%       Expected = any()
%%       Desc = string()
%%       Result = true | false
%% @doc Use an anonymous function to assert a pattern match.
fun_is(Fun, Expected, Desc) when is_function(Fun) ->
    is(Fun(Expected), true, Desc).

%% @equiv skip(TestFun, "")
skip(TestFun) when is_function(TestFun) ->
    skip(TestFun, "").

%% @spec skip(TestFun, Reason) -> ok
%%       TestFun = function()
%%       Reason = string()
%% @doc Skip a test.
skip(TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
    begin_skip(Reason),
    catch TestFun(),
    end_skip(),
    ok.

%% @spec skip(Q, TestFun, Reason) -> ok
%%       Q = true | false | function() 
%%       TestFun = function()
%%       Reason = string()
%% @doc Skips a test conditionally. The first argument to this function can
%% either be the 'true' or 'false' atoms or a function that returns 'true' or
%% 'false'.
skip(QFun, TestFun, Reason) when is_function(QFun), is_function(TestFun), is_list(Reason) ->
    case QFun() of
        true -> begin_skip(Reason), TestFun(), end_skip();
        _ -> TestFun()
    end,
    ok;

skip(Q, TestFun, Reason) when is_function(TestFun), is_list(Reason), Q == true ->
    begin_skip(Reason),
    TestFun(),
    end_skip(),
    ok;

skip(_, TestFun, Reason) when is_function(TestFun), is_list(Reason) ->
    TestFun(),
    ok.

%% @private
begin_skip(Reason) ->
    etap_server ! {self(), begin_skip, Reason}.

%% @private
end_skip() ->
    etap_server ! {self(), end_skip}.

% ---
% Internal / Private functions

%% @private
%% @doc Start the etap_server process if it is not running already.
ensure_test_server() ->
    case whereis(etap_server) of
        undefined ->
            proc_lib:start(?MODULE, start_etap_server,[]);
        _ ->
            diag("The test server is already running.")
    end.

%% @private
%% @doc Start the etap_server loop and register itself as the etap_server
%% process.
start_etap_server() ->
    catch register(etap_server, self()),
    proc_lib:init_ack(ok),
    etap:test_server(#test_state{
        planned = 0,
        count = 0,
        pass = 0,
        fail = 0,
        skip = 0,
        skip_reason = ""
    }).


%% @private
%% @doc The main etap_server receive/run loop. The etap_server receive loop
%% responds to seven messages apperatining to failure or passing of tests.
%% It is also used to initiate the testing process with the {_, plan, _}
%% message that clears the current test state.
test_server(State) ->
    NewState = receive
        {_From, plan, unknown} ->
            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
            State#test_state{
                planned = -1,
                count = 0,
                pass = 0,
                fail = 0,
                skip = 0,
                skip_reason = ""
            };
        {_From, plan, N} ->
            io:format("# Current time local ~s~n", [datetime(erlang:localtime())]),
            io:format("# Using etap version ~p~n", [ proplists:get_value(vsn, proplists:get_value(attributes, etap:module_info())) ]),
            io:format("1..~p~n", [N]),
            State#test_state{
                planned = N,
                count = 0,
                pass = 0,
                fail = 0,
                skip = 0,
                skip_reason = ""
            };
        {_From, begin_skip, Reason} ->
            State#test_state{
                skip = 1,
                skip_reason = Reason
            };
        {_From, end_skip} ->
            State#test_state{
                skip = 0,
                skip_reason = ""
            };
        {_From, pass, Desc} ->
            FullMessage = skip_diag(
                " - " ++ Desc,
                State#test_state.skip,
                State#test_state.skip_reason
            ),
            io:format("ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
            State#test_state{
                count = State#test_state.count + 1,
                pass = State#test_state.pass + 1
            };
            
        {_From, fail, Desc} ->
            FullMessage = skip_diag(
                " - " ++ Desc,
                State#test_state.skip,
                State#test_state.skip_reason
            ),
            io:format("not ok ~p ~s~n", [State#test_state.count + 1, FullMessage]),
            State#test_state{
                count = State#test_state.count + 1,
                fail = State#test_state.fail + 1
            };
        {From, state} ->
            From ! State,
            State;
        {_From, diag, Message} ->
            io:format("~s~n", [Message]),
            State;
        {From, count} ->
            From ! State#test_state.count,
            State;
        {From, is_skip} ->
            From ! State#test_state.skip,
            State;
        done ->
            exit(normal)
    end,
    test_server(NewState).

%% @private
%% @doc Process the result of a test and send it to the etap_server process.
mk_tap(Result, Desc) ->
    IsSkip = lib:sendw(etap_server, is_skip),
    case [IsSkip, Result] of
        [_, true] ->
            etap_server ! {self(), pass, Desc},
            true;                        
        [1, _] ->                        
            etap_server ! {self(), pass, Desc},
            true;                        
        _ ->                             
            etap_server ! {self(), fail, Desc},
            false
    end.

%% @private
%% @doc Format a date/time string.
datetime(DateTime) ->
    {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime,
    io_lib:format("~4.10.0B-~2.10.0B-~2.10.0B ~2.10.0B:~2.10.0B:~2.10.0B", [Year, Month, Day, Hour, Min, Sec]).

%% @private
%% @doc Craft an output message taking skip/todo into consideration.
skip_diag(Message, 0, _) ->
    Message;
skip_diag(_Message, 1, "") ->
    " # SKIP";
skip_diag(_Message, 1, Reason) ->
    " # SKIP : " ++ Reason.