From 598eee1cf5979b61171f623da7e3164244ca3792 Mon Sep 17 00:00:00 2001 From: Jan Lehnardt Date: Sun, 22 Feb 2009 13:50:38 +0000 Subject: Add runtime statistics -- without EUnit tests for now. git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@746691 13f79535-47bb-0310-9956-ffa450edef68 --- etc/couchdb/default.ini.tpl.in | 4 +++- share/Makefile.am | 1 - share/www/script/couch.js | 27 +++++++++++++++++++++++++++ share/www/script/couch_test_runner.js | 8 ++++++-- share/www/script/couch_tests.js | 1 - src/couchdb/Makefile.am | 8 +++++++- src/couchdb/couch_httpd.erl | 23 +++++++++++++++++++---- src/couchdb/couch_httpd_db.erl | 15 ++++++++++++--- src/couchdb/couch_httpd_misc_handlers.erl | 9 +-------- src/couchdb/couch_httpd_view.erl | 10 ++++++---- src/couchdb/couch_server.erl | 10 ++++++++-- src/couchdb/couch_server_sup.erl | 3 +-- 12 files changed, 90 insertions(+), 29 deletions(-) diff --git a/etc/couchdb/default.ini.tpl.in b/etc/couchdb/default.ini.tpl.in index e94e1437..0a5dd0a2 100644 --- a/etc/couchdb/default.ini.tpl.in +++ b/etc/couchdb/default.ini.tpl.in @@ -35,6 +35,8 @@ external_manager={couch_external_manager, start_link, []} db_update_notifier={couch_db_update_notifier_sup, start_link, []} query_servers={couch_query_servers, start_link, []} httpd={couch_httpd, start_link, []} +stats_aggregator={couch_stats_aggregator, start, []} +stats_collector={couch_stats_collector, start, []} [httpd_global_handlers] / = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} @@ -42,12 +44,12 @@ favicon.ico = {couch_httpd_misc_handlers, handle_favicon_req, "%localdatadir%/ww _utils = {couch_httpd_misc_handlers, handle_utils_dir_req, "%localdatadir%/www"} _all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} -_stats = {couch_httpd_misc_handlers, handle_stats_req} _active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} _config = {couch_httpd_misc_handlers, handle_config_req} _replicate = {couch_httpd_misc_handlers, handle_replicate_req} _uuids = {couch_httpd_misc_handlers, handle_uuids_req} _restart = {couch_httpd_misc_handlers, handle_restart_req} +_stats = {couch_httpd_stats_handlers, handle_stats_req} [httpd_db_handlers] _view = {couch_httpd_view, handle_view_req} diff --git a/share/Makefile.am b/share/Makefile.am index 40466cf9..476aa675 100644 --- a/share/Makefile.am +++ b/share/Makefile.am @@ -112,5 +112,4 @@ nobase_dist_localdata_DATA = \ www/script/test/purge.js \ www/script/test/config.js \ www/script/test/security_validation.js \ - www/script/test/max_dbs_open.js \ www/style/layout.css diff --git a/share/www/script/couch.js b/share/www/script/couch.js index 3f4db334..7bec5e32 100644 --- a/share/www/script/couch.js +++ b/share/www/script/couch.js @@ -312,6 +312,23 @@ CouchDB.request = function(method, uri, options) { return req; } +CouchDB.requestStats = function(module, key, aggregate, options) { + var options, optionsOrLast = Array.prototype.pop.apply(arguments); + if (typeof optionsOrLast == "string") { + options = null; + Array.prototype.push.apply(arguments, [optionsOrLast]); + } else { + options = optionsOrLast; + } + + var request_options = {}; + request_options.headers = {"Content-Type": "application/json"}; + + var stat = CouchDB.request("GET", "/_stats/" + Array.prototype.join.apply(arguments,["/"]) + (options ? + ("?" + CouchDB.params(options)) : ""), request_options).responseText; + return JSON.parse(stat)[module][key]; +} + CouchDB.uuids_cache = []; CouchDB.newUuids = function(n) { @@ -344,3 +361,13 @@ CouchDB.maybeThrowError = function(req) { throw result; } } + +CouchDB.params = function(options) { + options = options || {}; + var returnArray = []; + for(var key in options) { + var value = options[key]; + returnArray.push(key + "=" + value); + } + return returnArray.join("&"); +} \ No newline at end of file diff --git a/share/www/script/couch_test_runner.js b/share/www/script/couch_test_runner.js index ae357aeb..ae0dc573 100644 --- a/share/www/script/couch_test_runner.js +++ b/share/www/script/couch_test_runner.js @@ -152,13 +152,13 @@ function updateTestsFooter() { // display the line that failed. // Example: // T(MyValue==1); -function T(arg1, arg2) { +function T(arg1, arg2, testName) { if (!arg1) { if (currentRow) { if ($("td.details ol", currentRow).length == 0) { $("
    ").appendTo($("td.details", currentRow)); } - $("
  1. Assertion failed:
    2. ") + $("
  2. Assertion " + (testName ? "'" + testName + "'" : "") + " failed:
    3. ") .find("code").text((arg2 != null ? arg2 : arg1).toString()).end() .appendTo($("td.details ol", currentRow)); } @@ -166,6 +166,10 @@ function T(arg1, arg2) { } } +function TEquals(expected, actual, testName) { + T(equals(expected, actual), "expected '" + expected + "', got '" + actual + "'", testName); +} + function equals(a,b) { if (a === b) return true; try { diff --git a/share/www/script/couch_tests.js b/share/www/script/couch_tests.js index 7b25cb4f..a6195cf0 100644 --- a/share/www/script/couch_tests.js +++ b/share/www/script/couch_tests.js @@ -66,7 +66,6 @@ loadTest("compact.js"); loadTest("purge.js"); loadTest("config.js"); loadTest("security_validation.js"); -loadTest("max_dbs_open.js"); function makeDocs(start, end, templateDoc) { var templateDocSrc = templateDoc ? JSON.stringify(templateDoc) : "{}" diff --git a/src/couchdb/Makefile.am b/src/couchdb/Makefile.am index 8dd21256..10c06ebe 100644 --- a/src/couchdb/Makefile.am +++ b/src/couchdb/Makefile.am @@ -58,6 +58,7 @@ source_files = \ couch_httpd_show.erl \ couch_httpd_view.erl \ couch_httpd_misc_handlers.erl \ + couch_httpd_stats_handlers.erl \ couch_key_tree.erl \ couch_log.erl \ couch_os_process.erl \ @@ -66,6 +67,8 @@ source_files = \ couch_rep.erl \ couch_server.erl \ couch_server_sup.erl \ + couch_stats_aggregator.erl \ + couch_stats_collector.erl \ couch_stream.erl \ couch_task_status.erl \ couch_util.erl \ @@ -96,6 +99,7 @@ compiled_files = \ couch_httpd_show.beam \ couch_httpd_view.beam \ couch_httpd_misc_handlers.beam \ + couch_httpd_stats_handlers.beam \ couch_key_tree.beam \ couch_log.beam \ couch_os_process.beam \ @@ -104,6 +108,8 @@ compiled_files = \ couch_rep.beam \ couch_server.beam \ couch_server_sup.beam \ + couch_stats_aggregator.beam \ + couch_stats_collector.beam \ couch_stream.beam \ couch_task_status.beam \ couch_util.beam \ @@ -152,7 +158,7 @@ couch.app: couch.app.tpl # $(ERL) -noshell -run edoc_run files [\"$<\"] %.beam: %.erl couch_db.hrl - $(ERLC) $< + $(ERLC) ${TEST} $<; install-data-hook: if test -f "$(DESTDIR)/$(couchprivlibdir)/couch_erl_driver"; then \ diff --git a/src/couchdb/couch_httpd.erl b/src/couchdb/couch_httpd.erl index 5549e40a..9b175825 100644 --- a/src/couchdb/couch_httpd.erl +++ b/src/couchdb/couch_httpd.erl @@ -102,6 +102,7 @@ stop() -> handle_request(MochiReq, UrlHandlers, DbUrlHandlers) -> + statistics(runtime), % prepare request_time counter, see end of function AuthenticationFun = make_arity_1_fun( couch_config:get("httpd", "authentication_handler")), % for the path, use the raw path with the query string and fragment @@ -123,11 +124,8 @@ handle_request(MochiReq, UrlHandlers, DbUrlHandlers) -> mochiweb_headers:to_list(MochiReq:get(headers)) ]), - Method = + Method1 = case MochiReq:get(method) of - % alias HEAD to GET as mochiweb takes care of stripping the body - 'HEAD' -> 'GET'; - % already an atom Meth when is_atom(Meth) -> Meth; @@ -135,6 +133,15 @@ handle_request(MochiReq, UrlHandlers, DbUrlHandlers) -> % possible (if any module references the atom, then it's existing). Meth -> couch_util:to_existing_atom(Meth) end, + + increment_method_stats(Method1), + + % alias HEAD to GET as mochiweb takes care of stripping the body + Method = case Method1 of + 'HEAD' -> 'GET'; + Other -> Other + end, + HttpReq = #httpd{ mochi_req = MochiReq, method = Method, @@ -163,8 +170,14 @@ handle_request(MochiReq, UrlHandlers, DbUrlHandlers) -> RawUri, Resp:get(code) ]), + {_TotalRuntime, RequestTime} = statistics(runtime), + couch_stats_collector:record({couchdb, request_time}, RequestTime), + couch_stats_collector:increment({httpd, requests}), {ok, Resp}. +increment_method_stats(Method) -> + CounterName = list_to_atom(string:to_lower(atom_to_list(Method)) ++ "_requests"), + couch_stats_collector:increment({httpd, CounterName}). special_test_authentication_handler(Req) -> case header_value(Req, "WWW-Authenticate") of @@ -325,6 +338,7 @@ basic_username_pw(Req) -> start_chunked_response(#httpd{mochi_req=MochiReq}, Code, Headers) -> + couch_stats_collector:increment({http_status_codes, Code}), {ok, MochiReq:respond({Code, Headers ++ server_header(), chunked})}. send_chunk(Resp, Data) -> @@ -332,6 +346,7 @@ send_chunk(Resp, Data) -> {ok, Resp}. send_response(#httpd{mochi_req=MochiReq}, Code, Headers, Body) -> + couch_stats_collector:increment({http_status_codes, Code}), if Code >= 400 -> ?LOG_DEBUG("HTTPd ~p error response:~n ~s", [Code, Body]); true -> ok diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl index 95c96349..78339911 100644 --- a/src/couchdb/couch_httpd_db.erl +++ b/src/couchdb/couch_httpd_db.erl @@ -80,6 +80,7 @@ db_req(#httpd{method='POST',path_parts=[DbName]}=Req, Db) -> Doc = couch_doc:from_json_obj(couch_httpd:json_body(Req)), DocId = couch_util:new_uuid(), {ok, NewRev} = couch_db:update_doc(Db, Doc#doc{id=DocId, revs=[]}, []), + couch_stats_collector:increment({httpd, document_creates}), DocUrl = absolute_uri(Req, binary_to_list(<<"/",DbName/binary,"/",DocId/binary>>)), send_json(Req, 201, [{"Location", DocUrl}], {[ @@ -102,6 +103,7 @@ db_req(#httpd{path_parts=[_,<<"_ensure_full_commit">>]}=Req, _Db) -> send_method_not_allowed(Req, "POST"); db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>]}=Req, Db) -> + couch_stats_collector:increment({httpd, bulk_requests}), {JsonProps} = couch_httpd:json_body(Req), DocsArray = proplists:get_value(<<"docs">>, JsonProps), case couch_httpd:header_value(Req, "X-Couch-Full-Commit", "false") of @@ -377,6 +379,7 @@ db_doc_req(#httpd{method='DELETE'}=Req, Db, DocId) -> couch_httpd:send_error(Req, 409, <<"missing_rev">>, <<"Document rev/etag must be specified to delete">>); RevToDelete -> + couch_stats_collector:increment({httpd, document_deletes}), {ok, NewRev} = couch_db:delete_doc(Db, DocId, [RevToDelete]), send_json(Req, 200, {[ {ok, true}, @@ -391,6 +394,7 @@ db_doc_req(#httpd{method='GET'}=Req, Db, DocId) -> open_revs = Revs, options = Options } = parse_doc_query(Req), + couch_stats_collector:increment({httpd, document_reads}), case Revs of [] -> Doc = couch_doc_open(Db, DocId, Rev, Options), @@ -400,7 +404,7 @@ db_doc_req(#httpd{method='GET'}=Req, Db, DocId) -> [] -> [{"Etag", DiskEtag}]; % output etag only when we have no meta _ -> [] end, - send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) + send_json(Req, 200, Headers, couch_doc:to_json_obj(Doc, Options)) end); _ -> {ok, Results} = couch_db:open_doc_revs(Db, DocId, Revs, Options), @@ -467,8 +471,10 @@ db_doc_req(#httpd{method='PUT'}=Req, Db, DocId) -> end, case extract_header_rev(Req, ExplicitRev) of missing_rev -> + couch_stats_collector:increment({httpd, document_creates}), Revs = []; Rev -> + couch_stats_collector:increment({httpd, document_updates}), Revs = [Rev] end, {ok, NewRev} = couch_db:update_doc(Db, Doc#doc{id=DocId, revs=Revs}, Options), @@ -492,6 +498,7 @@ db_doc_req(#httpd{method='COPY'}=Req, Db, SourceDocId) -> % save new doc {ok, NewTargetRev} = couch_db:update_doc(Db, Doc#doc{id=TargetDocId, revs=TargetRev}, []), + couch_stats_collector:increment({httpd, document_copies}), send_json(Req, 201, [{"Etag", "\"" ++ binary_to_list(NewTargetRev) ++ "\""}], {[ {ok, true}, @@ -517,9 +524,9 @@ db_doc_req(#httpd{method='MOVE'}=Req, Db, SourceDocId) -> Doc#doc{id=TargetDocId, revs=TargetRev}, #doc{id=SourceDocId, revs=[SourceRev], deleted=true} ], - {ok, ResultRevs} = couch_db:update_docs(Db, Docs, []), - + couch_stats_collector:increment({httpd, document_moves}), + DocResults = lists:zipwith( fun(FDoc, NewRev) -> {[{id, FDoc#doc.id}, {rev, NewRev}]} @@ -622,8 +629,10 @@ db_attachment_req(#httpd{method=Method}=Req, Db, DocId, FileNameParts) Doc = case extract_header_rev(Req, couch_httpd:qs_value(Req, "rev")) of missing_rev -> % make the new doc + couch_stats_collector:increment({httpd, document_creates}), #doc{id=DocId}; Rev -> + couch_stats_collector:increment({httpd, document_updates}), case couch_db:open_doc_revs(Db, DocId, [Rev], []) of {ok, [{ok, Doc0}]} -> Doc0#doc{revs=[Rev]}; {ok, [Error]} -> throw(Error) diff --git a/src/couchdb/couch_httpd_misc_handlers.erl b/src/couchdb/couch_httpd_misc_handlers.erl index 92ff3b0a..be9e0033 100644 --- a/src/couchdb/couch_httpd_misc_handlers.erl +++ b/src/couchdb/couch_httpd_misc_handlers.erl @@ -14,7 +14,7 @@ -export([handle_welcome_req/2,handle_favicon_req/2,handle_utils_dir_req/2, handle_all_dbs_req/1,handle_replicate_req/1,handle_restart_req/1, - handle_uuids_req/1,handle_config_req/1,handle_stats_req/1, + handle_uuids_req/1,handle_config_req/1, handle_task_status_req/1]). -export([increment_update_seq_req/2]). @@ -63,13 +63,6 @@ handle_all_dbs_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). -handle_stats_req(#httpd{method='GET'}=Req) -> - ok = couch_httpd:verify_is_server_admin(Req), - send_json(Req, {couch_server:get_stats() ++ couch_file_stats:get_stats()}); -handle_stats_req(Req) -> - send_method_not_allowed(Req, "GET,HEAD"). - - handle_task_status_req(#httpd{method='GET'}=Req) -> ok = couch_httpd:verify_is_server_admin(Req), % convert the list of prop lists to a list of json objects diff --git a/src/couchdb/couch_httpd_view.erl b/src/couchdb/couch_httpd_view.erl index 9c8545cb..c4e2174e 100644 --- a/src/couchdb/couch_httpd_view.erl +++ b/src/couchdb/couch_httpd_view.erl @@ -28,8 +28,8 @@ design_doc_view(Req, Db, Id, ViewName, Keys) -> reduce = Reduce } = QueryArgs = parse_view_query(Req, Keys), DesignId = <<"_design/", Id/binary>>, - case couch_view:get_map_view(Db, DesignId, ViewName, Stale) of - {ok, View, Group} -> + Result = case couch_view:get_map_view(Db, DesignId, ViewName, Stale) of + {ok, View, Group} -> output_map_view(Req, View, Group, Db, QueryArgs, Keys); {not_found, Reason} -> case couch_view:get_reduce_view(Db, DesignId, ViewName, Stale) of @@ -45,7 +45,9 @@ design_doc_view(Req, Db, Id, ViewName, Keys) -> _ -> throw({not_found, Reason}) end - end. + end, + couch_stats_collector:increment({httpd, view_reads}), + Result. handle_view_req(#httpd{method='GET',path_parts=[_,_, Id, ViewName]}=Req, Db) -> design_doc_view(Req, Db, Id, ViewName, nil); @@ -60,7 +62,7 @@ handle_view_req(Req, _Db) -> handle_temp_view_req(#httpd{method='POST'}=Req, Db) -> QueryArgs = parse_view_query(Req), - + couch_stats_collector:increment({httpd, temporary_view_reads}), case couch_httpd:primary_header_value(Req, "content-type") of undefined -> ok; "application/json" -> ok; diff --git a/src/couchdb/couch_server.erl b/src/couchdb/couch_server.erl index 39b33c63..8f747a56 100644 --- a/src/couchdb/couch_server.erl +++ b/src/couchdb/couch_server.erl @@ -182,7 +182,9 @@ maybe_close_lru_db(#server{dbs_open=NumOpen, max_dbs_open=MaxOpen}=Server) maybe_close_lru_db(#server{dbs_open=NumOpen}=Server) -> % must free up the lru db. case try_close_lru(now()) of - ok -> {ok, Server#server{dbs_open=NumOpen-1}}; + ok -> + couch_stats_collector:decrement({couchdb, open_databases}), + {ok, Server#server{dbs_open=NumOpen - 1}}; Error -> Error end. @@ -235,6 +237,7 @@ handle_call({open, DbName, Options}, _From, Server) -> true = ets:insert(couch_dbs_by_pid, {MainPid, DbName}), true = ets:insert(couch_dbs_by_lru, {LruTime, DbName}), DbsOpen = Server2#server.dbs_open + 1, + couch_stats_collector:increment({couchdb, open_databases}), {reply, {ok, MainPid}, Server2#server{dbs_open=DbsOpen}}; Error -> @@ -270,6 +273,7 @@ handle_call({create, DbName, Options}, _From, Server) -> true = ets:insert(couch_dbs_by_pid, {MainPid, DbName}), true = ets:insert(couch_dbs_by_lru, {LruTime, DbName}), DbsOpen = Server2#server.dbs_open + 1, + couch_stats_collector:increment({couchdb, open_databases}), couch_db_update_notifier:notify({created, DbName}), {reply, {ok, MainPid}, Server2#server{dbs_open=DbsOpen}}; @@ -299,6 +303,7 @@ handle_call({delete, DbName, _Options}, _From, Server) -> true = ets:delete(couch_dbs_by_name, DbName), true = ets:delete(couch_dbs_by_pid, Pid), true = ets:delete(couch_dbs_by_lru, LruTime), + couch_stats_collector:decrement({couchdb, open_databases}), Server#server{dbs_open=Server#server.dbs_open - 1} end, case file:delete(FullFilepath) of @@ -328,6 +333,7 @@ handle_info({'EXIT', Pid, _Reason}, #server{dbs_open=DbsOpen}=Server) -> true = ets:delete(couch_dbs_by_pid, Pid), true = ets:delete(couch_dbs_by_name, DbName), true = ets:delete(couch_dbs_by_lru, LruTime), - {noreply, Server#server{dbs_open=DbsOpen-1}}; + couch_stats_collector:decrement({couchdb, open_databases}), + {noreply, Server#server{dbs_open=DbsOpen - 1}}; handle_info(Info, _Server) -> exit({unknown_message, Info}). diff --git a/src/couchdb/couch_server_sup.erl b/src/couchdb/couch_server_sup.erl index 627c34a9..34588cb8 100644 --- a/src/couchdb/couch_server_sup.erl +++ b/src/couchdb/couch_server_sup.erl @@ -158,7 +158,6 @@ start_primary_services() -> supervisor, dynamic}]}). - start_secondary_services() -> DaemonChildSpecs = [ begin @@ -173,7 +172,7 @@ start_secondary_services() -> end || {Name, SpecStr} <- couch_config:get("daemons"), SpecStr /= ""], - + supervisor:start_link({local, couch_secondary_services}, couch_server_sup, {{one_for_one, 10, 3600}, DaemonChildSpecs}). -- cgit v1.2.3