diff options
Diffstat (limited to 'provider_base')
23 files changed, 849 insertions, 42 deletions
diff --git a/provider_base/common.json b/provider_base/common.json index a4d9c5f2..649db0d9 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -25,9 +25,13 @@ "hosts": "=> hosts_file", "x509": { "use": true, + "use_commercial": false, "cert": "= x509.use ? file(:node_x509_cert, :missing => 'x509 certificate for node $node. Run `leap cert update`') : nil", "key": "= x509.use ? file(:node_x509_key, :missing => 'x509 key for node $node. Run `leap cert update`') : nil", - "ca_cert": "= try_file :ca_cert" + "ca_cert": "= try_file :ca_cert", + "commercial_cert": "= x509.use_commercial ? file([:commercial_cert, try{webapp.domain}||domain.full_suffix], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil", + "commercial_key": "= x509.use_commercial ? file([:commercial_key, try{webapp.domain}||domain.full_suffix], :missing => 'commercial x509 certificate for node $node. Add file $file, or run `leap cert csr` to generate a temporary self-signed cert and CSR you can use to purchase a real cert.') : nil", + "commercial_ca_cert": "= x509.use_commercial ? try_file(:commercial_ca_cert) : nil" }, "service_type": "internal_service", "development": { @@ -38,5 +42,13 @@ "enabled": true, "mail": { "smarthost": "= nodes_like_me[:services => :mx].exclude(self).field('domain.full')" + }, + "stunnel": { + "clients": {}, + "servers": {} + }, + "platform": { + "version": "= Leap::Platform.version.to_s", + "major_version": "= Leap::Platform.major_version" } } diff --git a/provider_base/files/service-definitions/v1/eip-service.json.erb b/provider_base/files/service-definitions/v1/eip-service.json.erb index 3b8976fd..4bd220df 100644 --- a/provider_base/files/service-definitions/v1/eip-service.json.erb +++ b/provider_base/files/service-definitions/v1/eip-service.json.erb @@ -42,8 +42,14 @@ end configuration = node.openvpn.configuration end - hsh["gateways"] = gateways.compact - hsh["locations"] = locations - hsh["openvpn_configuration"] = configuration + if gateways.any? + configuration = configuration.dup + if configuration['fragment'] && configuration['fragment'] == 1500 + configuration.delete('fragment') + end + hsh["gateways"] = gateways.compact + hsh["locations"] = locations + hsh["openvpn_configuration"] = configuration + end JSON.sorted_generate hsh %>
\ No newline at end of file diff --git a/provider_base/lib/macros.rb b/provider_base/lib/macros.rb new file mode 100644 index 00000000..ecc3e6ba --- /dev/null +++ b/provider_base/lib/macros.rb @@ -0,0 +1,15 @@ +# +# MACROS +# +# The methods in these files are available in the context of a .json configuration file. +# (The module LeapCli::Macro is included in Config::Object) +# + +require_relative 'macros/core' +require_relative 'macros/files' +require_relative 'macros/haproxy' +require_relative 'macros/hosts' +require_relative 'macros/keys' +require_relative 'macros/nodes' +require_relative 'macros/secrets' +require_relative 'macros/stunnel' diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb new file mode 100644 index 00000000..7de50f2f --- /dev/null +++ b/provider_base/lib/macros/core.rb @@ -0,0 +1,94 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + # + # return a fingerprint for a x509 certificate + # + def fingerprint(filename) + "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename)) + end + + # + # Creates a hash from the ssh key info in users directory, for use in + # updating authorized_keys file. Additionally, the 'monitor' public key is + # included, which is used by the monitor nodes to run particular commands + # remotely. + # + def authorized_keys + hash = {} + keys = Dir.glob(Path.named_path([:user_ssh, '*'])) + keys.sort.each do |keyfile| + ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ") + name = File.basename(File.dirname(keyfile)) + until hash[name].nil? + i ||= 1; name = "#{name}#{i+=1}" + end + hash[name] = { + "type" => ssh_type, + "key" => ssh_key + } + end + ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ") + hash[Leap::Platform.monitor_username] = { + "type" => ssh_type, + "key" => ssh_key + } + hash + end + + def assert(assertion) + if instance_eval(assertion) + true + else + raise AssertionFailed.new(assertion), assertion, caller + end + end + + def error(msg) + raise ConfigError.new(@node, msg), msg, caller + end + + # + # applies a JSON partial to this node + # + def apply_partial(partial_path) + manager.partials(partial_path).each do |partial_data| + self.deep_merge!(partial_data) + end + end + + # + # If at first you don't succeed, then it is time to give up. + # + # try{} returns nil if anything in the block throws an exception. + # + # You can wrap something that might fail in `try`, like so. + # + # "= try{ nodes[:services => 'tor'].first.ip_address } " + # + def try(&block) + yield + rescue NoMethodError + rescue ArgumentError + nil + end + + protected + + # + # returns a node list, if argument is not already one + # + def listify(node_list) + if node_list.is_a? Config::ObjectList + node_list + elsif node_list.is_a? Config::Object + Config::ObjectList.new(node_list) + else + raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller + end + end + + end +end diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb new file mode 100644 index 00000000..958958bc --- /dev/null +++ b/provider_base/lib/macros/files.rb @@ -0,0 +1,89 @@ +# encoding: utf-8 + +## +## FILES +## + +module LeapCli + module Macro + + # + # inserts the contents of a file + # + def file(filename, options={}) + if filename.is_a? Symbol + filename = [filename, @node.name] + end + filepath = Path.find_file(filename) + if filepath + if filepath =~ /\.erb$/ + ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding) + else + File.read(filepath, :encoding => 'UTF-8') + end + else + raise FileMissing.new(Path.named_path(filename), options) + "" + end + end + + # + # like #file, but allow missing files + # + def try_file(filename) + return file(filename) + rescue FileMissing + return nil + end + + # + # returns what the file path will be, once the file is rsynced to the server. + # an internal list of discovered file paths is saved, in order to rsync these files when needed. + # + # notes: + # + # * argument 'path' is relative to Path.provider/files or Path.provider_base/files + # * the path returned by this method is absolute + # * the path stored for use later by rsync is relative to Path.provider + # * if the path does not exist locally, but exists in provider_base, then the default file from + # provider_base is copied locally. this is required for rsync to work correctly. + # + def file_path(path, options={}) + if path.is_a? Symbol + path = [path, @node.name] + elsif path.is_a? String + # ensure it prefixed with files/ + unless path =~ /^files\// + path = "files/" + path + end + end + actual_path = Path.find_file(path) + if actual_path.nil? + if options[:missing] + raise FileMissing.new(Path.named_path(path), options) + else + Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file." + end + nil + else + if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/ + # if file is under Path.provider_base, we must copy the default file to + # to Path.provider in order for rsync to be able to sync the file. + local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider) + FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700 + FileUtils.install actual_path, local_provider_path, :mode => 0600 + Util.log :created, Path.relative_path(local_provider_path) + actual_path = local_provider_path + end + if File.directory?(actual_path) && actual_path !~ /\/$/ + actual_path += '/' # ensure directories end with /, important for building rsync command + end + relative_path = Path.relative_path(actual_path) + relative_path.sub!(/^files\//, '') # remove "files/" prefix + @node.file_paths << relative_path + File.join(Leap::Platform.files_dir, relative_path) + end + end + + end +end
\ No newline at end of file diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb new file mode 100644 index 00000000..602ae726 --- /dev/null +++ b/provider_base/lib/macros/haproxy.rb @@ -0,0 +1,73 @@ +# encoding: utf-8 + +## +## HAPROXY +## + +module LeapCli + module Macro + + # + # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. + # + # * node_list - a hash of nodes for the haproxy servers + # * stunnel_client - contains the mappings to local ports for each server node. + # * non_stunnel_port - in case self is included in node_list, the port to connect to. + # + # 1000 weight is used for nodes in the same location. + # 100 otherwise. + # + def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) + default_weight = 10 + local_weight = 100 + + # record the hosts_file + hostnames(node_list) + + # create a simple map for node name -> local stunnel accept port + accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| + name = stunnel_entry.first.sub /_[0-9]+$/, '' + hsh[name] = stunnel_entry.last['accept_port'] + hsh + end + + # if one the nodes in the node list is ourself, then there will not be a stunnel to it, + # but we need to include it anyway in the haproxy config. + if node_list[self.name] && non_stunnel_port + accept_ports[self.name] = non_stunnel_port + end + + # create the first pass of the servers hash + servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| + # make sure we have a port to talk to + unless accept_ports[node.name] + error "haproxy needs a local port to talk to when connecting to #{node.name}" + end + weight = default_weight + try { + weight = local_weight if self.location.name == node.location.name + } + hsh[node.name] = Config::Object[ + 'backup', false, + 'host', 'localhost', + 'port', accept_ports[node.name], + 'weight', weight + ] + if node.services.include?('couchdb') + hsh[node.name]['writable'] = node.couch.mode != 'mirror' + end + hsh + end + + # if there are some local servers, make the others backup + if servers.detect{|k,v| v.weight == local_weight} + servers.each do |k,server| + server['backup'] = server['weight'] == default_weight + end + end + + return servers + end + + end +end diff --git a/provider_base/lib/macros/hosts.rb b/provider_base/lib/macros/hosts.rb new file mode 100644 index 00000000..8281329f --- /dev/null +++ b/provider_base/lib/macros/hosts.rb @@ -0,0 +1,68 @@ +# encoding: utf-8 + +module LeapCli + module Macro + + ## + ## HOSTS + ## + + # + # records the list of hosts that are encountered for this node + # + def hostnames(nodes) + @referenced_nodes ||= Config::ObjectList.new + nodes = listify(nodes) + nodes.each_node do |node| + @referenced_nodes[node.name] ||= node + end + return nodes.values.collect {|node| node.domain.name} + end + + # + # Generates entries needed for updating /etc/hosts on a node (as a hash). + # + # Argument `nodes` can be nil or a list of nodes. If nil, only include the + # IPs of the other nodes this @node as has encountered (plus all mx nodes). + # + # Also, for virtual machines, we use the local address if this @node is in + # the same location as the node in question. + # + # We include the ssh public key for each host, so that the hash can also + # be used to generate the /etc/ssh/known_hosts + # + def hosts_file(nodes=nil) + if nodes.nil? + if @referenced_nodes && @referenced_nodes.any? + nodes = @referenced_nodes + nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes. + end + end + return {} unless nodes + hosts = {} + my_location = @node['location'] ? @node['location']['name'] : nil + nodes.each_node do |node| + hosts[node.name] = { + 'ip_address' => node.ip_address, + 'domain_internal' => node.domain.internal, + 'domain_full' => node.domain.full, + 'port' => node.ssh.port + } + node_location = node['location'] ? node['location']['name'] : nil + if my_location == node_location + if facts = @node.manager.facts[node.name] + if facts['ec2_public_ipv4'] + hosts[node.name]['ip_address'] = facts['ec2_public_ipv4'] + end + end + end + host_pub_key = Util::read_file([:node_ssh_pub_key,node.name]) + if host_pub_key + hosts[node.name]['host_pub_key'] = host_pub_key + end + end + hosts + end + + end +end
\ No newline at end of file diff --git a/provider_base/lib/macros/keys.rb b/provider_base/lib/macros/keys.rb new file mode 100644 index 00000000..0ed7ccd0 --- /dev/null +++ b/provider_base/lib/macros/keys.rb @@ -0,0 +1,83 @@ +# encoding: utf-8 + +# +# Macro for dealing with cryptographic keys +# + +module LeapCli + module Macro + + # + # return the path to the tor public key + # generating key if it is missing + # + def tor_public_key_path(path_name, key_type) + path = file_path(path_name) + if path.nil? + generate_tor_key(key_type) + file_path(path_name) + else + path + end + end + + # + # return the path to the tor private key + # generating key if it is missing + # + def tor_private_key_path(path_name, key_type) + path = file_path(path_name) + if path.nil? + generate_tor_key(key_type) + file_path(path_name) + else + path + end + end + + # + # Generates a onion_address from a public RSA key file. + # + # path_name is the named path of the Tor public key. + # + # Basically, an onion address is nothing more than a base32 encoding + # of the first 10 bytes of a sha1 digest of the public key. + # + # Additionally, Tor ignores the 22 byte header of the public key + # before taking the sha1 digest. + # + def onion_address(path_name) + require 'base32' + require 'base64' + require 'openssl' + path = Path.find_file([path_name, self.name]) + if path && File.exists?(path) + public_key_str = File.readlines(path).grep(/^[^-]/).join + public_key = Base64.decode64(public_key_str) + public_key = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header + sha1sum = Digest::SHA1.new.digest(public_key) + Base32.encode(sha1sum.slice(0,10)).downcase + else + LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path + end + end + + private + + def generate_tor_key(key_type) + if key_type == 'RSA' + require 'certificate_authority' + keypair = CertificateAuthority::MemoryKeyMaterial.new + bit_size = 1024 + LeapCli.log :generating, "%s bit RSA Tor key" % bit_size do + keypair.generate_key(bit_size) + LeapCli::Util.write_file! [:node_tor_priv_key, self.name], keypair.private_key.to_pem + LeapCli::Util.write_file! [:node_tor_pub_key, self.name], keypair.public_key.to_pem + end + else + LeapCli.bail! 'tor.key.type of %s is not yet supported' % key_type + end + end + + end +end diff --git a/provider_base/lib/macros/nodes.rb b/provider_base/lib/macros/nodes.rb new file mode 100644 index 00000000..0c6668a0 --- /dev/null +++ b/provider_base/lib/macros/nodes.rb @@ -0,0 +1,88 @@ +# encoding: utf-8 + +## +## node related macros +## + +module LeapCli + module Macro + + # + # the list of all the nodes + # + def nodes + global.nodes + end + + # + # grab an environment appropriate provider + # + def provider + global.env(@node.environment).provider + end + + # + # returns a list of nodes that match the same environment + # + # if @node.environment is not set, we return other nodes + # where environment is not set. + # + def nodes_like_me + nodes[:environment => @node.environment] + end + + # + # returns a list of nodes that match the location name + # and environment of @node. + # + def nodes_near_me + if @node['location'] && @node['location']['name'] + nodes_like_me['location.name' => @node.location.name] + else + nodes_like_me['location' => nil] + end + end + + # + # + # picks a node out from the node list in such a way that: + # + # (1) which nodes picked which nodes is saved in secrets.json + # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node + # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly. + # + # if the node_list is empty, an exception is raised. + # if node_list size is 1, then that node is returned and nothing is + # memorized via the secrets.json file. + # + # `label` is needed to distinguish between pools of nodes for different purposes. + # + # TODO: more evenly balance after all the nodes have been picked. + # + def pick_node(label, node_list) + if node_list.any? + if node_list.size == 1 + return node_list.values.first + else + secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})" + secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {} + secrets_value[@node.name] ||= begin + node_to_pick = nil + node_list.each_node do |node| + next if secrets_value.values.include?(node.name) + node_to_pick = node.name + end + node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one. + node_to_pick + end + picked_node_name = secrets_value[@node.name] + @manager.secrets.set(secrets_key, secrets_value, @node.environment) + return node_list[picked_node_name] + end + else + raise ArgumentError.new('pick_node(node_list): node_list cannot be empty') + end + end + + end +end
\ No newline at end of file diff --git a/provider_base/lib/macros/secrets.rb b/provider_base/lib/macros/secrets.rb new file mode 100644 index 00000000..51bf3971 --- /dev/null +++ b/provider_base/lib/macros/secrets.rb @@ -0,0 +1,39 @@ +# encoding: utf-8 + +require 'base32' + +module LeapCli + module Macro + + # + # inserts a named secret, generating it if needed. + # + # manager.export_secrets should be called later to capture any newly generated secrets. + # + # +length+ is the character length of the generated password. + # + def secret(name, length=32) + @manager.secrets.set(name, Util::Secret.generate(length), @node[:environment]) + end + + # inserts a base32 encoded secret + def base32_secret(name, length=20) + @manager.secrets.set(name, Base32.encode(Util::Secret.generate(length)), @node[:environment]) + end + + # Picks a random obfsproxy port from given range + def rand_range(name, range) + @manager.secrets.set(name, rand(range), @node[:environment]) + end + + # + # inserts an hexidecimal secret string, generating it if needed. + # + # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4) + # + def hex_secret(name, bit_length=128) + @manager.secrets.set(name, Util::Secret.generate_hex(bit_length), @node[:environment]) + end + + end +end
\ No newline at end of file diff --git a/provider_base/lib/macros/stunnel.rb b/provider_base/lib/macros/stunnel.rb new file mode 100644 index 00000000..f16308c7 --- /dev/null +++ b/provider_base/lib/macros/stunnel.rb @@ -0,0 +1,95 @@ +## +## STUNNEL +## + +# +# About stunnel +# -------------------------- +# +# The network looks like this: +# +# From the client's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# consumer app -> localhost:accept_port -> connect:connect_port -> ?? +# +# From the server's perspective: +# +# |------- stunnel client --------------| |---------- stunnel server -----------------------| +# ?? -> *:accept_port -> localhost:connect_port -> service +# + +module LeapCli + module Macro + + # + # stunnel configuration for the client side. + # + # +node_list+ is a ObjectList of nodes running stunnel servers. + # + # +port+ is the real port of the ultimate service running on the servers + # that the client wants to connect to. + # + # * accept_port is the port on localhost to which local clients + # can connect. it is auto generated serially. + # + # * connect_port is the port on the stunnel server to connect to. + # it is auto generated from the +port+ argument. + # + # generates an entry appropriate to be passed directly to + # create_resources(stunnel::service, hiera('..'), defaults) + # + # local ports are automatically generated, starting at 4000 + # and incrementing in sorted order (by node name). + # + def stunnel_client(node_list, port, options={}) + @next_stunnel_port ||= 4000 + node_list = listify(node_list) + hostnames(node_list) # record the hosts + result = Config::ObjectList.new + node_list.each_node do |node| + if node.name != self.name || options[:include_self] + result["#{node.name}_#{port}"] = Config::Object[ + 'accept_port', @next_stunnel_port, + 'connect', node.domain.internal, + 'connect_port', stunnel_port(port), + 'original_port', port + ] + @next_stunnel_port += 1 + end + end + result + end + + # + # generates a stunnel server entry. + # + # +port+ is the real port targeted service. + # + # * `accept_port` is the publicly bound port + # * `connect_port` is the port that the local service is running on. + # + def stunnel_server(port) + { + "accept_port" => stunnel_port(port), + "connect_port" => port + } + end + + private + + # + # maps a real port to a stunnel port (used as the connect_port in the client config + # and the accept_port in the server config) + # + def stunnel_port(port) + port = port.to_i + if port < 50000 + return port + 10000 + else + return port - 10000 + end + end + + end +end
\ No newline at end of file diff --git a/provider_base/provider.json b/provider_base/provider.json index 743964ee..77437935 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -1,6 +1,6 @@ { "domain": "REQUIRED", - "domain_internal": "= domain.sub(/\\..*$/,'.i')", + "domain_internal": "= domain.sub(/\\.[^\\.]*$/, '.i')", "name": { "en": "REQUIRED" }, @@ -44,7 +44,7 @@ "digest": "SHA256", "life_span": "10y", "server_certificates": { - "bit_size": 2048, + "bit_size": 4096, "digest": "SHA256", "life_span": "1y" }, @@ -56,7 +56,6 @@ "unlimited_prefix": "UNLIMITED" } }, - "hiera_sync_destination": "/etc/leap", "client_version": { "min": "0.5", "max": null diff --git a/provider_base/services/_couchdb_master.json b/provider_base/services/_couchdb_master.json new file mode 100644 index 00000000..20c6f99b --- /dev/null +++ b/provider_base/services/_couchdb_master.json @@ -0,0 +1,8 @@ +// +// Applied to master couchdb node when there is a single master +// +{ + "couch": { + "mode": "master" + } +}
\ No newline at end of file diff --git a/provider_base/services/_couchdb_mirror.json b/provider_base/services/_couchdb_mirror.json new file mode 100644 index 00000000..6a3402bd --- /dev/null +++ b/provider_base/services/_couchdb_mirror.json @@ -0,0 +1,21 @@ +// +// Applied to all non-master couchdb nodes +// +{ + "stunnel": { + "clients": { + "couch_client": "= stunnel_client(nodes[couch.replication.masters.keys], couch.port)" + } + }, + "couch": { + "mode": "mirror", + "replication": { + // for now, pick the first close one, or the first one. + // in the future, maybe use haproxy to balance among all the masters + "masters": "= try{pick_node(:couch_master,nodes_near_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')} || try{pick_node(:couch_master,nodes_like_me['services' => 'couchdb']['couch.master' => true]).pick_fields('domain.internal', 'couch.port')}", + "username": "replication", + "password": "= secret :couch_replication_password", + "role": "replication" + } + } +} diff --git a/provider_base/services/_couchdb_multimaster.json b/provider_base/services/_couchdb_multimaster.json new file mode 100644 index 00000000..0f340e00 --- /dev/null +++ b/provider_base/services/_couchdb_multimaster.json @@ -0,0 +1,24 @@ +// +// Only applied to master couchdb nodes when there are multiple masters +// +{ + "stunnel": { + "servers": { + "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", + "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)" + }, + "clients": { + "epmd_clients": "= stunnel_client(nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'], couch.bigcouch.epmd_port)", + "ednp_clients": "= stunnel_client(nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'], couch.bigcouch.ednp_port)" + } + }, + "couch": { + "mode": "multimaster", + "bigcouch": { + "epmd_port": 4369, + "ednp_port": 9002, + "cookie": "= secret :bigcouch_cookie", + "neighbors": "= nodes_like_me['services' => 'couchdb']['couch.mode' => 'multimaster'].exclude(self).field('domain.full')" + } + } +} diff --git a/provider_base/services/couchdb.json b/provider_base/services/couchdb.json index 5f1b5381..8b1386f8 100644 --- a/provider_base/services/couchdb.json +++ b/provider_base/services/couchdb.json @@ -3,20 +3,13 @@ "use": true }, "stunnel": { - "couch_server": "= stunnel_server(couch.port)", - "epmd_server": "= stunnel_server(couch.bigcouch.epmd_port)", - "epmd_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.bigcouch.epmd_port)", - "ednp_server": "= stunnel_server(couch.bigcouch.ednp_port)", - "ednp_clients": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.bigcouch.ednp_port)" + "servers": { + "couch_server": "= stunnel_server(couch.port)" + } }, "couch": { + "master": false, "port": 5984, - "bigcouch": { - "epmd_port": 4369, - "ednp_port": 9002, - "cookie": "= secret :bigcouch_cookie", - "neighbors": "= nodes_like_me[:services => :couchdb].exclude(self).field('domain.full')" - }, "users": { "admin": { "username": "admin", @@ -47,10 +40,15 @@ "username": "webapp", "password": "= secret :couch_webapp_password", "salt": "= hex_secret :couch_webapp_password_salt, 128" + }, + "replication": { + "username": "replication", + "password": "= secret :couch_replication_password", + "salt": "= hex_secret :couch_replication_password_salt, 128" } }, - "webapp": { - "nagios_test_pw": "= secret :nagios_test_password" + "webapp": { + "nagios_test_pw": "= secret :nagios_test_password" } } } diff --git a/provider_base/services/couchdb.rb b/provider_base/services/couchdb.rb new file mode 100644 index 00000000..3bee3a67 --- /dev/null +++ b/provider_base/services/couchdb.rb @@ -0,0 +1,60 @@ +####################################################################### +### +### NOTE! +### +### Currently, mirrors do not work! The only thing that works is all +### nodes multimaster or a single master. +### +####################################################################### +# +# custom logic for couchdb json resolution +# ============================================ +# +# There are three modes for a node: +# +# Multimaster +# ----------- +# +# Multimaster uses bigcouch (soon to use couchdb in replication mode +# similar to bigcouch). +# +# Use "multimaster" mode when: +# +# * multiple nodes are marked couch.master +# * OR no nodes are marked couch.master +# +# Master +# ------ +# +# Master uses plain couchdb that is readable and writable. +# +# Use "master" mode when: +# +# * Exactly one node, this one, is marked as master. +# +# Mirror +# ------ +# +# Mirror creates a read-only copy of the database. It uses plain coucdhb +# with legacy couchdb replication (http based). +# +# This does not currently work, because http replication can't handle +# the number of user databases. +# +# Use "mirror" mode when: +# +# * some nodes are marked couch.master +# * AND this node is not a master +# + +master_count = nodes_like_me['services' => 'couchdb']['couch.master' => true].size + +if master_count == 0 + apply_partial 'services/_couchdb_multimaster.json' +elsif couch.master && master_count > 1 + apply_partial 'services/_couchdb_multimaster.json' +elsif couch.master && master_count == 1 + apply_partial 'services/_couchdb_master.json' +else + apply_partial 'services/_couchdb_mirror.json' +end diff --git a/provider_base/services/monitor.json b/provider_base/services/monitor.json index 03f6c6d1..10d5ac81 100644 --- a/provider_base/services/monitor.json +++ b/provider_base/services/monitor.json @@ -1,7 +1,9 @@ { "nagios": { "nagiosadmin_pw": "= secret :nagios_admin_password", - "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('domain.internal', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" + "domains_internal": "= nagios.hosts.values.map{|h|h['domain_internal_suffix']}.uniq", + "environments": "= Hash[ nagios.hosts.values.map{|h|h['environment']}.uniq.map{|e| [e||'default',{'contact_emails'=>global.env(e).provider.contacts.default}]} ]", + "hosts": "= (self.environment == 'local' ? nodes_like_me : nodes[:environment => '!local']).pick_fields('environment', 'domain.internal', 'domain.internal_suffix', 'domain.full_suffix', 'ip_address', 'services', 'openvpn.gateway_address', 'ssh.port')" }, "hosts": "= self.environment == 'local' ? hosts_file(nodes_like_me) : hosts_file(nodes[:environment => '!local'])", "ssh": { @@ -12,11 +14,9 @@ }, "x509": { "use": true, + "use_commercial": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", - "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'", - "commercial_cert": "= file [:commercial_cert, domain.full_suffix]", - "commercial_key": "= file [:commercial_key, domain.full_suffix]", - "commercial_ca_cert": "= try_file :commercial_ca_cert" + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" } } diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index 731dee9a..11293ae8 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -1,9 +1,14 @@ { "stunnel": { - "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + "clients": { + "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + } }, "haproxy": { - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client)" + "couch": { + "listen_port": 4096, + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" + } }, "couchdb_leap_mx_user": { "username": "= global.services[:couchdb].couch.users[:leap_mx].username", @@ -13,12 +18,10 @@ "mynetworks": "= nodes['environment' => '!local'].map{|name, n| [n.ip_address, (global.facts[name]||{})['ec2_public_ipv4']]}.flatten.compact.uniq", "x509": { "use": true, + "use_commercial": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", - "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'", - "commercial_cert": "= file [:commercial_cert, domain.full_suffix]", - "commercial_key": "= file [:commercial_key, domain.full_suffix]", - "commercial_ca_cert": "= try_file :commercial_ca_cert" + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" }, "service_type": "user_service" } diff --git a/provider_base/services/obfsproxy.json b/provider_base/services/obfsproxy.json new file mode 100644 index 00000000..979d0ef9 --- /dev/null +++ b/provider_base/services/obfsproxy.json @@ -0,0 +1,9 @@ +{ + "obfsproxy": { + "scramblesuit": { + "password": "= base32_secret('scramblesuit_password_'+name)", + "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" + }, + "gateway_address": "= try{pick_node(:obfs_gateway,nodes_near_me['services' => 'openvpn']).pick_fields('openvpn.gateway_address')} || try{pick_node(:obfs_gateway,nodes_like_me['services' => 'openvpn']).pick_fields('openvpn.gateway_address')}" + } +} diff --git a/provider_base/services/openvpn.json b/provider_base/services/openvpn.json index 090afcd6..11cb0dc2 100644 --- a/provider_base/services/openvpn.json +++ b/provider_base/services/openvpn.json @@ -24,7 +24,15 @@ "auth": "SHA1", "cipher": "AES-128-CBC", "keepalive": "10 30", - "tun-ipv6": true + "tun-ipv6": true, + "fragment": 1500 } + }, + "obfsproxy": { + "scramblesuit": { + "password": "= base32_secret('scramblesuit_password_'+name)", + "port" : "= rand_range('scramblesuit_port_'+name, 18000..32000)" + }, + "gateway_address": "= openvpn.gateway_address" } } diff --git a/provider_base/services/tor.json b/provider_base/services/tor.json index fc365a19..55d3d2ee 100644 --- a/provider_base/services/tor.json +++ b/provider_base/services/tor.json @@ -3,6 +3,13 @@ "bandwidth_rate": 6550, "contacts": "= [provider.contacts['tor'] || provider.contacts.default].flatten", "nickname": "= (self.name + secret(:tor_family)).sub('_','')[0..18]", - "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')" + "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')", + "hidden_service": { + "active": null, + "key_type": "RSA", + "public_key": "= tor_public_key_path(:node_tor_pub_key, tor.hidden_service.key_type) if tor.hidden_service.active", + "private_key": "= tor_private_key_path(:node_tor_priv_key, tor.hidden_service.key_type) if tor.hidden_service.active", + "address": "= onion_address(:node_tor_pub_key) if tor.hidden_service.active" + } } } diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index bbb52094..67744f99 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -1,6 +1,7 @@ { "webapp": { "admins": [], + "forbidden_usernames": ["admin", "administrator", "arin-admin", "certmaster", "contact", "info", "maildrop", "postmaster", "ssladmin", "www-data"], "domain": "= domain.full_suffix", "modules": ["user", "billing", "help"], "couchdb_webapp_user": { @@ -13,6 +14,7 @@ "allow_limited_certs": "= provider.service.allow_limited_bandwidth", "allow_unlimited_certs": "= provider.service.allow_unlimited_bandwidth", "allow_anonymous_certs": "= provider.service.allow_anonymous", + "allow_registration": "= provider.service.allow_registration", "default_service_level": "= provider.service.default_service_level", "service_levels": "= provider.service.levels", "secret_token": "= secret :webapp_secret_token", @@ -20,19 +22,27 @@ "secure": false, "git": { "source": "https://leap.se/git/leap_web", - "revision": "origin/master" + "revision": "origin/version/0.6" }, "client_version": "= provider.client_version", "nagios_test_user": { "username": "nagios_test", "password": "= secret :nagios_test_password" - } + }, + "engines": [ + "support" + ] }, "stunnel": { - "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + "clients": { + "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" + } }, "haproxy": { - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.couch_client, global.services[:couchdb].couch.port)" + "couch": { + "listen_port": 4096, + "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" + } }, "definition_files": { "provider": "= file :provider_json_template", @@ -59,11 +69,9 @@ }, "x509": { "use": true, + "use_commercial": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", - "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", - "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'", - "commercial_cert": "= file [:commercial_cert, webapp.domain]", - "commercial_key": "= file [:commercial_key, webapp.domain]", - "commercial_ca_cert": "= try_file :commercial_ca_cert" + "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`.'", + "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`.'" } } |