try to create the db many times until it exists

5 files changed, 95 insertions, 37 deletions

diff --git a/lib/tapicero/user_database.rb b/lib/tapicero/user_database.rb
index 7c621ae..14526ce 100644
--- a/lib/tapicero/user_database.rb
+++ b/lib/tapicero/user_database.rb
@@ -9,19 +9,32 @@ module Tapicero
       @db = couch.database(config.options[:db_prefix] + user_id)
     end
 
+    #
+    # Create the user db, and keep trying until successful.
+    #
     def create
-      retry_request_once "Creating database" do
-        create_db
+      retry_until "creating database", method(:exists?) do
+        begin
+          couch.create_db(db.name)
+        rescue RestClient::PreconditionFailed
+          # silently eat preconditionfailed, since it might be bogus.
+          # we will keep trying until db actually exists.
+        end
       end
     end
 
+    #
+    # upload security document
+    #
     def secure(security = nil)
       security ||= config.options[:security]
       # let's not overwrite if we have a security doc already
       return if secured? && !Tapicero::FLAGS.include?('--overwrite-security')
       retry_request_once "Writing security to" do
-        Tapicero.logger.debug security.to_json
-        CouchRest.put security_url, security
+        ignore_conflicts do
+          Tapicero.logger.debug security.to_json
+          CouchRest.put security_url, security
+        end
       end
     end
 
@@ -49,16 +62,16 @@ module Tapicero
     def upload_design_doc(file)
       old = CouchRest.get design_url(file.basename('.json'))
     rescue RestClient::ResourceNotFound
-      begin
+      ignore_conflicts do
         CouchRest.put design_url(file.basename('.json')), JSON.parse(file.read)
-      rescue RestClient::Conflict
       end
     end
 
-
     def destroy
       retry_request_once "Deleting database" do
-        delete_db
+        ignore_not_found do
+          db.delete! if db
+        end
       end
     end
 
@@ -66,32 +79,11 @@ module Tapicero
       db.name
     end
 
-    protected
+    private
 
     #
-    # Sometimes attempting to create the db will fail with PreconditionFailed.
-    # This error is supposed to be returned only when the db already exists.
-    # However, sometimes the db will be created and PreconditionFailed will
-    # be returned.
-    #
-    # Might throw one of the following exceptions:
-    # * RestClient::BadRequest
-    # * RestClient::Unauthorized
+    # If at first you don't succeed, try one more time.
     #
-    def create_db
-      db.info # test if db exists
-    rescue RestClient::ResourceNotFound
-      begin
-        couch.create_db(db.name)
-      rescue RestClient::PreconditionFailed
-      end
-    end
-
-    def delete_db
-      db.delete! if db
-    rescue RestClient::ResourceNotFound  # no database found
-    end
-
     def retry_request_once(action)
       second_try ||= false
       Tapicero.logger.debug "#{action} #{db.name}"
@@ -108,6 +100,55 @@ module Tapicero
       end
     end
 
+    #
+    # most of the time, we can safely ignore conflicts. It just
+    # means that another tapicero daemon beat us to the punch.
+    #
+    def ignore_conflicts
+      yield
+    rescue RestClient::Conflict => exc
+      raise exc if Tapicero::RERAISE
+    end
+
+    def ignore_not_found
+      yield
+    rescue RestClient::ResourceNotFound
+    end
+
+    #
+    # captures and logs any uncaught rest client Exceptions
+    #
+    def log_rest_client_errors(msg)
+      yield
+    rescue RestClient::Exception => exc
+      raise exc if Tapicero::RERAISE
+      log_error "#{msg} #{db.name} failed due to: ", exc
+    end
+
+    #
+    # keeps trying block until method returns true.
+    # gives up after 100 tries
+    #
+    def retry_until(msg, method)
+      tries = 0
+      while(true)
+        tries += 1
+        if tries > 100
+          Tapicero.logger.error "Gave up: #{msg}"
+          break
+        else
+          log_rest_client_errors msg do
+            yield
+          end
+        end
+        if method.call()
+          break
+        else
+          sleep 1
+        end
+      end
+    end
+
     def log_error(message, exc)
       # warn message is a one liner so nagios can parse it
       Tapicero.logger.warn message.to_s + exc.class.name + ': ' + exc.to_s
@@ -124,6 +165,13 @@ module Tapicero
       end
     end
 
+    def exists?
+      db.info
+      return true
+    rescue RestClient::ResourceNotFound
+      return false
+    end
+
     def security_url
       db.root + "/_security"
     end
diff --git a/lib/tapicero_daemon.rb b/lib/tapicero_daemon.rb
index 9eb342b..86f924e 100644
--- a/lib/tapicero_daemon.rb
+++ b/lib/tapicero_daemon.rb
@@ -11,11 +11,17 @@ require 'tapicero/user_event_handler'
 module Tapicero
   module Daemon
     while true
-      users = CouchRest::Changes.new('users')
-      UserEventHandler.new(users)
-      users.listen
-      Tapicero.logger.info('Lost contact with couchdb, will try again in 10 seconds')
-      sleep 10
+      begin
+        users = CouchRest::Changes.new('users')
+        UserEventHandler.new(users)
+        users.listen
+        Tapicero.logger.info('Lost contact with couchdb, will try again in 10 seconds')
+        sleep 10
+      rescue SystemCallError => exc
+        Tapicero.logger.info('Problem connecting to couchdb (#{exc}). Will try again in 10 seconds.')
+        sleep 10
+        retry
+      end
     end
   end
 end
diff --git a/test/integration/actions_test.rb b/test/integration/actions_test.rb
index f3d3faa..a975e0b 100644
--- a/test/integration/actions_test.rb
+++ b/test/integration/actions_test.rb
@@ -5,6 +5,7 @@ class ActionsTest < Tapicero::IntegrationTest
   def setup
     TapiceroProcess.run_with_config("test/config.yaml")
     create_user
+    sleep 0.1
   end
 
   def teardown
diff --git a/test/support/tapicero_process.rb b/test/support/tapicero_process.rb
index fdfb37d..ab246dd 100644
--- a/test/support/tapicero_process.rb
+++ b/test/support/tapicero_process.rb
@@ -24,7 +24,7 @@ module TapiceroProcess
           puts "bin/tapicero run -- '#{config_path}'"
           exec "bin/tapicero run -- '#{config_path}'"
         else
-          exec "bin/tapicero run -- '#{config_path}' > /dev/null 2>&1"
+          exec "bin/tapicero run -- '#{config_path}' > /dev/null | grep -v ^tapicero:"
         end
       end
       Process.detach(other_process)
diff --git a/test/test_helper.rb b/test/test_helper.rb
index ec42f41..31eaa2b 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -33,3 +33,6 @@ MiniTest.after_run {
   TapiceroProcess.kill!
 }
 
+puts
+puts "   REMINDER: check /tmp/tapicero.log for errors"
+puts
\ No newline at end of file