From 75e62f1aa97b6702b82d28ddc768a92c0a831b5c Mon Sep 17 00:00:00 2001 From: Azul Date: Thu, 19 Jun 2014 20:04:38 +0200 Subject: first stab at replicating couch nodes from should probably be a local port and we should use stunnels to connect to the master --- manifests/mirror_db.pp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 manifests/mirror_db.pp diff --git a/manifests/mirror_db.pp b/manifests/mirror_db.pp new file mode 100644 index 0000000..c2e5b21 --- /dev/null +++ b/manifests/mirror_db.pp @@ -0,0 +1,16 @@ +define couchdb::mirror_db ( + $host='127.0.0.1:5984', + $from='', + $to='' ) +{ + $source = "$from$name" + $target = "$to$name" + + couchdb::document { "${name}_replication": + db => "_replicate", + id => "${name}_replication", + host => $host, + data => "{ \"source\": ${source}, \"target\": ${target}, \"continuous\": true }", + require => Couchdb::Query["create_db_${name}"] + } +} -- cgit v1.2.3 From 630ae0d3a9e80c189f485b3e79fbbe0aecd21d7c Mon Sep 17 00:00:00 2001 From: Azul Date: Fri, 20 Jun 2014 18:08:02 +0200 Subject: allow setting custom netrc in query and document defines We need to perform some actions as the replication user for plain couch. --- files/couch-doc-update | 10 ++++++---- manifests/document.pp | 7 ++++--- manifests/mirror_db.pp | 12 ++++++++---- manifests/query.pp | 8 ++++++-- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/files/couch-doc-update b/files/couch-doc-update index 5ceed5e..fea6456 100644 --- a/files/couch-doc-update +++ b/files/couch-doc-update @@ -62,6 +62,7 @@ def process_options doc_id = nil new_data = nil filename = nil + netrc_file = nil loop do case ARGV[0] when '--host' then ARGV.shift; host = ARGV.shift @@ -69,6 +70,7 @@ def process_options when '--id' then ARGV.shift; doc_id = ARGV.shift when '--data' then ARGV.shift; new_data = ARGV.shift when '--file' then ARGV.shift; filename = ARGV.shift + when '--netrc-file' then ARGV.shift; netrc_file = ARGV.shift when /^-/ then usage("Unknown option: #{ARGV[0].inspect}") else break end @@ -76,7 +78,7 @@ def process_options usage("Missing required option") unless db_name && doc_id && new_data new_data = MultiJson.load(new_data) new_data.merge!(read_file(filename)) if filename - db = CouchRest.database(connection_string(db_name, host)) + db = CouchRest.database(connection_string(db_name, host, netrc_file)) return db, doc_id, new_data end @@ -124,14 +126,14 @@ def create_document(db, doc_id, data) db.save_doc(data) end -def connection_string(database, host) +def connection_string(database, host, netrc_file = nil) protocol = "http" #hostname = "127.0.0.1" port = "5984" username = "admin" password = "" - netrc = File.read('/etc/couchdb/couchdb.netrc') + netrc = File.read(netrc_file || '/etc/couchdb/couchdb.netrc') netrc.scan(/\w+ [\w\.]+/).each do |key_value| key, value = key_value.split ' ' case key @@ -148,7 +150,7 @@ end def usage(s) $stderr.puts(s) - $stderr.puts("Usage: #{File.basename($0)} --host --db --id --data [--file ]") + $stderr.puts("Usage: #{File.basename($0)} --host --db --id --data [--file ] [--netrc-file ]") exit(2) end diff --git a/manifests/document.pp b/manifests/document.pp index 1153296..3ecc05d 100644 --- a/manifests/document.pp +++ b/manifests/document.pp @@ -10,6 +10,7 @@ define couchdb::document( $id, $host = '127.0.0.1:5984', $data = '{}', + $netrc = '/etc/couchdb/couchdb.netrc', $ensure = 'content') { $url = "${host}/${db}/${id}" @@ -17,7 +18,7 @@ define couchdb::document( case $ensure { default: { err ( "unknown ensure value '${ensure}'" ) } content: { - exec { "couch-doc-update --host ${host} --db ${db} --id ${id} --data \'${data}\'": + exec { "couch-doc-update --netrc-file ${netrc} --host ${host} --db ${db} --id ${id} --data \'${data}\'": require => Exec['wait_for_couchdb'], unless => "couch-doc-diff $url '$data'" } @@ -28,7 +29,7 @@ define couchdb::document( cmd => 'PUT', host => $host, path => "${db}/${id}", - unless => "/usr/bin/curl -s -f --netrc-file /etc/couchdb/couchdb.netrc ${url}" + unless => "/usr/bin/curl -s -f --netrc-file ${netrc} ${url}" } } @@ -37,7 +38,7 @@ define couchdb::document( cmd => 'DELETE', host => $host, path => "${db}/${id}", - unless => "/usr/bin/curl -s -f --netrc-file /etc/couchdb/couchdb.netrc ${url}" + unless => "/usr/bin/curl -s -f --netrc-file ${netrc} ${url}" } } } diff --git a/manifests/mirror_db.pp b/manifests/mirror_db.pp index c2e5b21..06007e0 100644 --- a/manifests/mirror_db.pp +++ b/manifests/mirror_db.pp @@ -3,14 +3,18 @@ define couchdb::mirror_db ( $from='', $to='' ) { - $source = "$from$name" - $target = "$to$name" + $source = "$from/$name" + if $to == '' { $target = $name } + else { $target = "$to/$name" } + + $replication_user = "replication" + $replication_role = "replication" couchdb::document { "${name}_replication": - db => "_replicate", + db => "_replicator", id => "${name}_replication", host => $host, - data => "{ \"source\": ${source}, \"target\": ${target}, \"continuous\": true }", + data => "{ \"source\": \"${source}\", \"target\": \"${target}\", \"continuous\": true, \"user_ctx\": { \"name\": \"${replication_user}\", \"roles\": [\"${replication_role}\"] } }", require => Couchdb::Query["create_db_${name}"] } } diff --git a/manifests/query.pp b/manifests/query.pp index 72ae4d2..9507ca1 100644 --- a/manifests/query.pp +++ b/manifests/query.pp @@ -1,7 +1,11 @@ define couchdb::query ( - $cmd, $path, $host='127.0.0.1:5984', $data = '{}', $unless = undef) { + $cmd, $path, + $netrc='/etc/couchdb/couchdb.netrc', + $host='127.0.0.1:5984', + $data = '{}', + $unless = undef) { - exec { "/usr/bin/curl -s --netrc-file /etc/couchdb/couchdb.netrc -X ${cmd} ${host}/${path} --data \'${data}\'": + exec { "/usr/bin/curl -s --netrc-file ${netrc} -X ${cmd} ${host}/${path} --data \'${data}\'": require => [ Package['curl'], Exec['wait_for_couchdb'] ], unless => $unless } -- cgit v1.2.3 From 6a2f8b50a0374d467662d9c3325c7229468f607d Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 22:06:39 +0200 Subject: update couch-doc-update to also filter _replication fields we only need to keep id and rev to prevent conflicts --- files/couch-doc-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/couch-doc-update b/files/couch-doc-update index fea6456..df216db 100644 --- a/files/couch-doc-update +++ b/files/couch-doc-update @@ -116,7 +116,7 @@ def get_document(db, doc_id) end def update_document(db, doc, data) - doc.reject! {|k,v| !k.start_with? '_'} + doc.reject! {|k,v| !["_id", "_rev"].include? k} doc.merge! data db.save_doc(doc) end -- cgit v1.2.3 From 15092303c4828331457e7cebcae470de7ea7f3c7 Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 22:07:13 +0200 Subject: create netrc files for all users --- manifests/add_user.pp | 6 ++++++ manifests/query/setup.pp | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/manifests/add_user.pp b/manifests/add_user.pp index 1d25fd5..29c6a8c 100644 --- a/manifests/add_user.pp +++ b/manifests/add_user.pp @@ -30,4 +30,10 @@ define couchdb::add_user ( $roles, $pw, $salt = '' ) { id => "org.couchdb.user:${name}", data => $data } + + couchdb::query::setup { $name: + user => $name, + pw => $pw, + } + } diff --git a/manifests/query/setup.pp b/manifests/query/setup.pp index b68e54f..451eb53 100644 --- a/manifests/query/setup.pp +++ b/manifests/query/setup.pp @@ -1,6 +1,6 @@ define couchdb::query::setup ($user, $pw, $host='127.0.0.1') { - file { '/etc/couchdb/couchdb.netrc': + file { "/etc/couchdb/couchdb-${user}.netrc": content => "machine ${host} login ${user} password ${pw}", mode => '0600', owner => $::couchdb::base::couchdb_user, -- cgit v1.2.3 From 1ecb96caeb9cf236a4aa94baa8ea3a079d178cae Mon Sep 17 00:00:00 2001 From: Azul Date: Mon, 23 Jun 2014 22:07:58 +0200 Subject: use replication user for creating the replications --- manifests/mirror_db.pp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/manifests/mirror_db.pp b/manifests/mirror_db.pp index 06007e0..a6ba549 100644 --- a/manifests/mirror_db.pp +++ b/manifests/mirror_db.pp @@ -1,20 +1,21 @@ define couchdb::mirror_db ( $host='127.0.0.1:5984', $from='', - $to='' ) + $to='', + $user='replication', + $role='replication' + ) { $source = "$from/$name" if $to == '' { $target = $name } else { $target = "$to/$name" } - $replication_user = "replication" - $replication_role = "replication" - couchdb::document { "${name}_replication": - db => "_replicator", - id => "${name}_replication", - host => $host, - data => "{ \"source\": \"${source}\", \"target\": \"${target}\", \"continuous\": true, \"user_ctx\": { \"name\": \"${replication_user}\", \"roles\": [\"${replication_role}\"] } }", + db => "_replicator", + id => "${name}_replication", + netrc => "/etc/couchdb/couchdb-${user}.netrc", + host => $host, + data => "{ \"source\": \"${source}\", \"target\": \"${target}\", \"continuous\": true, \"user_ctx\": { \"name\": \"${user}\", \"roles\": [\"${role}\"] } }", require => Couchdb::Query["create_db_${name}"] } } -- cgit v1.2.3 From 14f863f54c46bdf6f681c8a5ba660aba6a7419eb Mon Sep 17 00:00:00 2001 From: elijah Date: Wed, 25 Jun 2014 18:02:35 -0700 Subject: minor lint --- manifests/mirror_db.pp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/manifests/mirror_db.pp b/manifests/mirror_db.pp index a6ba549..7615c65 100644 --- a/manifests/mirror_db.pp +++ b/manifests/mirror_db.pp @@ -6,16 +6,16 @@ define couchdb::mirror_db ( $role='replication' ) { - $source = "$from/$name" + $source = "${from}/${name}" if $to == '' { $target = $name } - else { $target = "$to/$name" } + else { $target = "${to}/${name}" } couchdb::document { "${name}_replication": - db => "_replicator", - id => "${name}_replication", - netrc => "/etc/couchdb/couchdb-${user}.netrc", - host => $host, - data => "{ \"source\": \"${source}\", \"target\": \"${target}\", \"continuous\": true, \"user_ctx\": { \"name\": \"${user}\", \"roles\": [\"${role}\"] } }", + db => "_replicator", + id => "${name}_replication", + netrc => "/etc/couchdb/couchdb-${user}.netrc", + host => $host, + data => "{ \"source\": \"${source}\", \"target\": \"${target}\", \"continuous\": true, \"user_ctx\": { \"name\": \"${user}\", \"roles\": [\"${role}\"] } }", require => Couchdb::Query["create_db_${name}"] } } -- cgit v1.2.3