use CouchRest::Database#changes, store and retrieve seq

This commit bundles a few different changes:
* we now use the CouchRest::Database#changes stream instead of our own
* we store and read the last sequence number so on a restart we can start where we left
* we now have a UserDatabase class rather than the CouchDatabaseCreator. It also knows about it's name and we create an instance of it per database we want to create. It's also more flexible when the databases already exist.

9 files changed, 108 insertions, 42 deletions

diff --git a/config/default.yaml b/config/default.yaml
index 3a08497..0e58e6e 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -21,3 +21,5 @@ security:
     names: 
       - soledad
     roles: []
+seq_file: "/tmp/tapicero.seq"
+log_file: "/var/log/leap/tapicero.log"
diff --git a/lib/tapicero.rb b/lib/tapicero.rb
index 7287934..967a9a7 100644
--- a/lib/tapicero.rb
+++ b/lib/tapicero.rb
@@ -15,7 +15,7 @@ Tapicero::Config.load(BASE_DIR, 'config/default.yaml', LEAP_CA_CONFIG, ARGV.grep
 #
 # Load Tapicero
 #
-require 'tapicero/json_stream'
-require 'tapicero/couch_stream'
+# require 'tapicero/json_stream'
+# require 'tapicero/couch_stream'
 require 'tapicero/couch_changes'
-require 'tapicero/couch_database_creator'
+require 'tapicero/user_database'
diff --git a/lib/tapicero/config.rb b/lib/tapicero/config.rb
index 6e0fdfe..ad8076a 100644
--- a/lib/tapicero/config.rb
+++ b/lib/tapicero/config.rb
@@ -8,6 +8,8 @@ module Tapicero
     attr_accessor :db_prefix
     attr_accessor :couch_connection
     attr_accessor :security
+    attr_accessor :seq_file
+    attr_accessor :log_file
 
     def self.load(base_dir, *configs)
       configs.each do |file_path|
diff --git a/lib/tapicero/couch_changes.rb b/lib/tapicero/couch_changes.rb
index abb451d..3b7da59 100644
--- a/lib/tapicero/couch_changes.rb
+++ b/lib/tapicero/couch_changes.rb
@@ -1,7 +1,21 @@
+require 'couchrest'
+require 'fileutils'
+
 module Tapicero
   class CouchChanges
-    def initialize(stream)
-      @stream = stream
+
+    attr_accessor :db
+
+    def initialize(db, seq_filename)
+      @db = db
+      @seq_filename = seq_filename
+      FileUtils.touch(seq_filename)
+      unless File.writable?(seq_filename)
+        raise StandardError.new("Can't access sequence file")
+      end
+      @since = File.read(seq_filename)
+    rescue Errno::ENOENT => e
+      puts "No sequence file found. Starting from scratch"
     end
 
     def created(hash = {}, &block)
@@ -12,24 +26,41 @@ module Tapicero
       end
     end
 
-    def last_seq
-      @stream.get "_changes", :limit => 1, :descending => true do |hash|
-        return hash[:last_seq]
-      end
-    end
-
     def listen
-      @stream.get "_changes", :feed => :continuous, :since => last_seq do |hash|
+      puts "listening..."
+      puts "Starting at sequence #{since}"
+      db.changes :feed => :continuous, :since => since, :heartbeat => 1000 do |hash|
         callbacks(hash)
       end
     end
 
+    protected
+
+    def since
+      @since ||= 0  # fetch_last_seq
+    end
+
     def callbacks(hash)
-      #changed
-      return if hash[:deleted]
-      return unless changes = hash[:changes]
-      return created(hash) if changes[0][:rev].start_with?('1-')
-      #updated
+      #changed callback
+      return if hash["deleted"] # deleted_callback
+      return unless changes = hash["changes"]
+      created(hash) if changes[0]["rev"].start_with?('1-')
+      store_seq(hash["seq"])
+      #updated callback
+    end
+
+    def store_seq(seq)
+      File.write(@seq_filename, seq)
     end
+
+    #
+    # UNUSED: this is useful for only following new sequences.
+    #
+    def fetch_last_seq
+      hash = db.changes :limit => 1, :descending => true
+      puts "starting at seq: " + hash["last_seq"]
+      return hash["last_seq"]
+    end
+
   end
 end
diff --git a/lib/tapicero/couch_database_creator.rb b/lib/tapicero/couch_database_creator.rb
deleted file mode 100644
index 4d39c1b..0000000
--- a/lib/tapicero/couch_database_creator.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-require 'couchrest'
-require 'json'
-
-module Tapicero
-  class CouchDatabaseCreator
-
-    def initialize(host)
-      @host = host
-      @couch = CouchRest.new(host)
-    end
-
-    def create(name, security = {})
-      db = @couch.create_db(name)
-      puts security.to_json
-      puts "-> #{@host}/#{name}/_security"
-      CouchRest.put "#{@host}/#{name}/_security", security
-    end
-  end
-end
diff --git a/lib/tapicero/couch_stream.rb b/lib/tapicero/couch_stream.rb
index 8583010..b63a9fd 100644
--- a/lib/tapicero/couch_stream.rb
+++ b/lib/tapicero/couch_stream.rb
@@ -1,3 +1,8 @@
+#
+# UNUSED: we currently use CouchRest's streamer instead. Still keeping this
+#         around because it's a simple alternative that works.
+#
+
 module Tapicero
   class CouchStream
     def initialize(database_url)
diff --git a/lib/tapicero/json_stream.rb b/lib/tapicero/json_stream.rb
index d906ce2..64b160f 100644
--- a/lib/tapicero/json_stream.rb
+++ b/lib/tapicero/json_stream.rb
@@ -2,8 +2,11 @@ require 'net/http'
 require 'uri'
 require 'yajl'
 
-
+# UNUSED: We're currently using couchrest instead as that is what we use all
+#         over the place. It internally uses curl to fetch the stream.
+#
 # Since Yajl HTTP Stream will go a way in version 2.0 here's a simple substitude.
+#
 module Tapicero
   class JsonStream
 
diff --git a/lib/tapicero/user_database.rb b/lib/tapicero/user_database.rb
new file mode 100644
index 0000000..fcdd272
--- /dev/null
+++ b/lib/tapicero/user_database.rb
@@ -0,0 +1,39 @@
+require 'couchrest'
+require 'json'
+
+module Tapicero
+  class UserDatabase
+
+    def initialize(host, name)
+      @host = host
+      @name = name
+    end
+
+    def create
+      begin
+        CouchRest.new(host).create_db(name)
+      rescue RestClient::PreconditionFailed  # database already existed
+      end
+    end
+
+    def secure(security)
+      # let's not overwrite if we have a security doc already
+      return if secured?
+      puts security.to_json
+      puts "-> #{security_url}"
+      CouchRest.put security_url, security
+    end
+
+    protected
+
+    def secured?
+      CouchRest.get(security_url).keys.any?
+    end
+
+    def security_url
+      "#{host}/#{name}/_security"
+    end
+
+    attr_reader :host, :name
+  end
+end
diff --git a/lib/tapicero_daemon.rb b/lib/tapicero_daemon.rb
index 52e60e3..a5e41eb 100644
--- a/lib/tapicero_daemon.rb
+++ b/lib/tapicero_daemon.rb
@@ -10,12 +10,15 @@ require 'tapicero'
 module Tapicero
   puts    " * Observing #{Config.couch_host_without_password}"
   puts    " * Tracking #{Config.users_db_name}"
-  stream   = CouchStream.new(Config.couch_host + '/' + Config.users_db_name)
-  users = CouchChanges.new(stream)
-  creator = CouchDatabaseCreator.new(Config.couch_host)
+  # stream   = CouchStream.new(Config.couch_host + '/' + Config.users_db_name)
+  db = CouchRest.new(Config.couch_host).database(Config.users_db_name)
+  users = CouchChanges.new(db, Config.seq_file)
+
   users.created do |hash|
-    puts "Created user " + hash[:id]
-    creator.create(Config.db_prefix + hash[:id], Config.security)
+    puts "Created user " + hash['id']
+    db = UserDatabase.new(Config.couch_host, Config.db_prefix + hash['id'])
+    db.create
+    db.secure(Config.security)
   end
 
   users.listen