From 97c460cd1133c18fd82a649253db755a38bea0ed Mon Sep 17 00:00:00 2001 From: elijah Date: Sun, 14 Dec 2014 18:56:14 -0800 Subject: try to create the db many times until it exists --- lib/tapicero/user_database.rb | 110 ++++++++++++++++++++++++++++----------- lib/tapicero_daemon.rb | 16 ++++-- test/integration/actions_test.rb | 1 + test/support/tapicero_process.rb | 2 +- test/test_helper.rb | 3 ++ 5 files changed, 95 insertions(+), 37 deletions(-) diff --git a/lib/tapicero/user_database.rb b/lib/tapicero/user_database.rb index 7c621ae..14526ce 100644 --- a/lib/tapicero/user_database.rb +++ b/lib/tapicero/user_database.rb @@ -9,19 +9,32 @@ module Tapicero @db = couch.database(config.options[:db_prefix] + user_id) end + # + # Create the user db, and keep trying until successful. + # def create - retry_request_once "Creating database" do - create_db + retry_until "creating database", method(:exists?) do + begin + couch.create_db(db.name) + rescue RestClient::PreconditionFailed + # silently eat preconditionfailed, since it might be bogus. + # we will keep trying until db actually exists. + end end end + # + # upload security document + # def secure(security = nil) security ||= config.options[:security] # let's not overwrite if we have a security doc already return if secured? && !Tapicero::FLAGS.include?('--overwrite-security') retry_request_once "Writing security to" do - Tapicero.logger.debug security.to_json - CouchRest.put security_url, security + ignore_conflicts do + Tapicero.logger.debug security.to_json + CouchRest.put security_url, security + end end end @@ -49,16 +62,16 @@ module Tapicero def upload_design_doc(file) old = CouchRest.get design_url(file.basename('.json')) rescue RestClient::ResourceNotFound - begin + ignore_conflicts do CouchRest.put design_url(file.basename('.json')), JSON.parse(file.read) - rescue RestClient::Conflict end end - def destroy retry_request_once "Deleting database" do - delete_db + ignore_not_found do + db.delete! if db + end end end @@ -66,32 +79,11 @@ module Tapicero db.name end - protected + private # - # Sometimes attempting to create the db will fail with PreconditionFailed. - # This error is supposed to be returned only when the db already exists. - # However, sometimes the db will be created and PreconditionFailed will - # be returned. - # - # Might throw one of the following exceptions: - # * RestClient::BadRequest - # * RestClient::Unauthorized + # If at first you don't succeed, try one more time. # - def create_db - db.info # test if db exists - rescue RestClient::ResourceNotFound - begin - couch.create_db(db.name) - rescue RestClient::PreconditionFailed - end - end - - def delete_db - db.delete! if db - rescue RestClient::ResourceNotFound # no database found - end - def retry_request_once(action) second_try ||= false Tapicero.logger.debug "#{action} #{db.name}" @@ -108,6 +100,55 @@ module Tapicero end end + # + # most of the time, we can safely ignore conflicts. It just + # means that another tapicero daemon beat us to the punch. + # + def ignore_conflicts + yield + rescue RestClient::Conflict => exc + raise exc if Tapicero::RERAISE + end + + def ignore_not_found + yield + rescue RestClient::ResourceNotFound + end + + # + # captures and logs any uncaught rest client Exceptions + # + def log_rest_client_errors(msg) + yield + rescue RestClient::Exception => exc + raise exc if Tapicero::RERAISE + log_error "#{msg} #{db.name} failed due to: ", exc + end + + # + # keeps trying block until method returns true. + # gives up after 100 tries + # + def retry_until(msg, method) + tries = 0 + while(true) + tries += 1 + if tries > 100 + Tapicero.logger.error "Gave up: #{msg}" + break + else + log_rest_client_errors msg do + yield + end + end + if method.call() + break + else + sleep 1 + end + end + end + def log_error(message, exc) # warn message is a one liner so nagios can parse it Tapicero.logger.warn message.to_s + exc.class.name + ': ' + exc.to_s @@ -124,6 +165,13 @@ module Tapicero end end + def exists? + db.info + return true + rescue RestClient::ResourceNotFound + return false + end + def security_url db.root + "/_security" end diff --git a/lib/tapicero_daemon.rb b/lib/tapicero_daemon.rb index 9eb342b..86f924e 100644 --- a/lib/tapicero_daemon.rb +++ b/lib/tapicero_daemon.rb @@ -11,11 +11,17 @@ require 'tapicero/user_event_handler' module Tapicero module Daemon while true - users = CouchRest::Changes.new('users') - UserEventHandler.new(users) - users.listen - Tapicero.logger.info('Lost contact with couchdb, will try again in 10 seconds') - sleep 10 + begin + users = CouchRest::Changes.new('users') + UserEventHandler.new(users) + users.listen + Tapicero.logger.info('Lost contact with couchdb, will try again in 10 seconds') + sleep 10 + rescue SystemCallError => exc + Tapicero.logger.info('Problem connecting to couchdb (#{exc}). Will try again in 10 seconds.') + sleep 10 + retry + end end end end diff --git a/test/integration/actions_test.rb b/test/integration/actions_test.rb index f3d3faa..a975e0b 100644 --- a/test/integration/actions_test.rb +++ b/test/integration/actions_test.rb @@ -5,6 +5,7 @@ class ActionsTest < Tapicero::IntegrationTest def setup TapiceroProcess.run_with_config("test/config.yaml") create_user + sleep 0.1 end def teardown diff --git a/test/support/tapicero_process.rb b/test/support/tapicero_process.rb index fdfb37d..ab246dd 100644 --- a/test/support/tapicero_process.rb +++ b/test/support/tapicero_process.rb @@ -24,7 +24,7 @@ module TapiceroProcess puts "bin/tapicero run -- '#{config_path}'" exec "bin/tapicero run -- '#{config_path}'" else - exec "bin/tapicero run -- '#{config_path}' > /dev/null 2>&1" + exec "bin/tapicero run -- '#{config_path}' > /dev/null | grep -v ^tapicero:" end end Process.detach(other_process) diff --git a/test/test_helper.rb b/test/test_helper.rb index ec42f41..31eaa2b 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -33,3 +33,6 @@ MiniTest.after_run { TapiceroProcess.kill! } +puts +puts " REMINDER: check /tmp/tapicero.log for errors" +puts \ No newline at end of file -- cgit v1.2.3