summaryrefslogtreecommitdiff
path: root/provider_base/lib
diff options
context:
space:
mode:
Diffstat (limited to 'provider_base/lib')
-rw-r--r--provider_base/lib/macros.rb15
-rw-r--r--provider_base/lib/macros/core.rb94
-rw-r--r--provider_base/lib/macros/files.rb89
-rw-r--r--provider_base/lib/macros/haproxy.rb73
-rw-r--r--provider_base/lib/macros/hosts.rb68
-rw-r--r--provider_base/lib/macros/keys.rb83
-rw-r--r--provider_base/lib/macros/nodes.rb88
-rw-r--r--provider_base/lib/macros/secrets.rb39
-rw-r--r--provider_base/lib/macros/stunnel.rb95
9 files changed, 0 insertions, 644 deletions
diff --git a/provider_base/lib/macros.rb b/provider_base/lib/macros.rb
deleted file mode 100644
index ecc3e6ba..00000000
--- a/provider_base/lib/macros.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-#
-# MACROS
-#
-# The methods in these files are available in the context of a .json configuration file.
-# (The module LeapCli::Macro is included in Config::Object)
-#
-
-require_relative 'macros/core'
-require_relative 'macros/files'
-require_relative 'macros/haproxy'
-require_relative 'macros/hosts'
-require_relative 'macros/keys'
-require_relative 'macros/nodes'
-require_relative 'macros/secrets'
-require_relative 'macros/stunnel'
diff --git a/provider_base/lib/macros/core.rb b/provider_base/lib/macros/core.rb
deleted file mode 100644
index 7de50f2f..00000000
--- a/provider_base/lib/macros/core.rb
+++ /dev/null
@@ -1,94 +0,0 @@
-# encoding: utf-8
-
-module LeapCli
- module Macro
-
- #
- # return a fingerprint for a x509 certificate
- #
- def fingerprint(filename)
- "SHA256: " + X509.fingerprint("SHA256", Path.named_path(filename))
- end
-
- #
- # Creates a hash from the ssh key info in users directory, for use in
- # updating authorized_keys file. Additionally, the 'monitor' public key is
- # included, which is used by the monitor nodes to run particular commands
- # remotely.
- #
- def authorized_keys
- hash = {}
- keys = Dir.glob(Path.named_path([:user_ssh, '*']))
- keys.sort.each do |keyfile|
- ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ")
- name = File.basename(File.dirname(keyfile))
- until hash[name].nil?
- i ||= 1; name = "#{name}#{i+=1}"
- end
- hash[name] = {
- "type" => ssh_type,
- "key" => ssh_key
- }
- end
- ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ")
- hash[Leap::Platform.monitor_username] = {
- "type" => ssh_type,
- "key" => ssh_key
- }
- hash
- end
-
- def assert(assertion)
- if instance_eval(assertion)
- true
- else
- raise AssertionFailed.new(assertion), assertion, caller
- end
- end
-
- def error(msg)
- raise ConfigError.new(@node, msg), msg, caller
- end
-
- #
- # applies a JSON partial to this node
- #
- def apply_partial(partial_path)
- manager.partials(partial_path).each do |partial_data|
- self.deep_merge!(partial_data)
- end
- end
-
- #
- # If at first you don't succeed, then it is time to give up.
- #
- # try{} returns nil if anything in the block throws an exception.
- #
- # You can wrap something that might fail in `try`, like so.
- #
- # "= try{ nodes[:services => 'tor'].first.ip_address } "
- #
- def try(&block)
- yield
- rescue NoMethodError
- rescue ArgumentError
- nil
- end
-
- protected
-
- #
- # returns a node list, if argument is not already one
- #
- def listify(node_list)
- if node_list.is_a? Config::ObjectList
- node_list
- elsif node_list.is_a? Config::Object
- Config::ObjectList.new(node_list)
- else
- raise ArgumentError, 'argument must be a node or node list, not a `%s`' % node_list.class, caller
- end
- end
-
- end
-end
diff --git a/provider_base/lib/macros/files.rb b/provider_base/lib/macros/files.rb
deleted file mode 100644
index 958958bc..00000000
--- a/provider_base/lib/macros/files.rb
+++ /dev/null
@@ -1,89 +0,0 @@
-# encoding: utf-8
-
-##
-## FILES
-##
-
-module LeapCli
- module Macro
-
- #
- # inserts the contents of a file
- #
- def file(filename, options={})
- if filename.is_a? Symbol
- filename = [filename, @node.name]
- end
- filepath = Path.find_file(filename)
- if filepath
- if filepath =~ /\.erb$/
- ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding)
- else
- File.read(filepath, :encoding => 'UTF-8')
- end
- else
- raise FileMissing.new(Path.named_path(filename), options)
- ""
- end
- end
-
- #
- # like #file, but allow missing files
- #
- def try_file(filename)
- return file(filename)
- rescue FileMissing
- return nil
- end
-
- #
- # returns what the file path will be, once the file is rsynced to the server.
- # an internal list of discovered file paths is saved, in order to rsync these files when needed.
- #
- # notes:
- #
- # * argument 'path' is relative to Path.provider/files or Path.provider_base/files
- # * the path returned by this method is absolute
- # * the path stored for use later by rsync is relative to Path.provider
- # * if the path does not exist locally, but exists in provider_base, then the default file from
- # provider_base is copied locally. this is required for rsync to work correctly.
- #
- def file_path(path, options={})
- if path.is_a? Symbol
- path = [path, @node.name]
- elsif path.is_a? String
- # ensure it prefixed with files/
- unless path =~ /^files\//
- path = "files/" + path
- end
- end
- actual_path = Path.find_file(path)
- if actual_path.nil?
- if options[:missing]
- raise FileMissing.new(Path.named_path(path), options)
- else
- Util::log 2, :skipping, "file_path(\"#{path}\") because there is no such file."
- end
- nil
- else
- if actual_path =~ /^#{Regexp.escape(Path.provider_base)}/
- # if file is under Path.provider_base, we must copy the default file to
- # to Path.provider in order for rsync to be able to sync the file.
- local_provider_path = actual_path.sub(/^#{Regexp.escape(Path.provider_base)}/, Path.provider)
- FileUtils.mkdir_p File.dirname(local_provider_path), :mode => 0700
- FileUtils.install actual_path, local_provider_path, :mode => 0600
- Util.log :created, Path.relative_path(local_provider_path)
- actual_path = local_provider_path
- end
- if File.directory?(actual_path) && actual_path !~ /\/$/
- actual_path += '/' # ensure directories end with /, important for building rsync command
- end
- relative_path = Path.relative_path(actual_path)
- relative_path.sub!(/^files\//, '') # remove "files/" prefix
- @node.file_paths << relative_path
- File.join(Leap::Platform.files_dir, relative_path)
- end
- end
-
- end
-end \ No newline at end of file
diff --git a/provider_base/lib/macros/haproxy.rb b/provider_base/lib/macros/haproxy.rb
deleted file mode 100644
index 602ae726..00000000
--- a/provider_base/lib/macros/haproxy.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# encoding: utf-8
-
-##
-## HAPROXY
-##
-
-module LeapCli
- module Macro
-
- #
- # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to.
- #
- # * node_list - a hash of nodes for the haproxy servers
- # * stunnel_client - contains the mappings to local ports for each server node.
- # * non_stunnel_port - in case self is included in node_list, the port to connect to.
- #
- # 1000 weight is used for nodes in the same location.
- # 100 otherwise.
- #
- def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil)
- default_weight = 10
- local_weight = 100
-
- # record the hosts_file
- hostnames(node_list)
-
- # create a simple map for node name -> local stunnel accept port
- accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry|
- name = stunnel_entry.first.sub /_[0-9]+$/, ''
- hsh[name] = stunnel_entry.last['accept_port']
- hsh
- end
-
- # if one the nodes in the node list is ourself, then there will not be a stunnel to it,
- # but we need to include it anyway in the haproxy config.
- if node_list[self.name] && non_stunnel_port
- accept_ports[self.name] = non_stunnel_port
- end
-
- # create the first pass of the servers hash
- servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node|
- # make sure we have a port to talk to
- unless accept_ports[node.name]
- error "haproxy needs a local port to talk to when connecting to #{node.name}"
- end
- weight = default_weight
- try {
- weight = local_weight if self.location.name == node.location.name
- }
- hsh[node.name] = Config::Object[
- 'backup', false,
- 'host', 'localhost',
- 'port', accept_ports[node.name],
- 'weight', weight
- ]
- if node.services.include?('couchdb')
- hsh[node.name]['writable'] = node.couch.mode != 'mirror'
- end
- hsh
- end
-
- # if there are some local servers, make the others backup
- if servers.detect{|k,v| v.weight == local_weight}
- servers.each do |k,server|
- server['backup'] = server['weight'] == default_weight
- end
- end
-
- return servers
- end
-
- end
-end
diff --git a/provider_base/lib/macros/hosts.rb b/provider_base/lib/macros/hosts.rb
deleted file mode 100644
index 8281329f..00000000
--- a/provider_base/lib/macros/hosts.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-# encoding: utf-8
-
-module LeapCli
- module Macro
-
- ##
- ## HOSTS
- ##
-
- #
- # records the list of hosts that are encountered for this node
- #
- def hostnames(nodes)
- @referenced_nodes ||= Config::ObjectList.new
- nodes = listify(nodes)
- nodes.each_node do |node|
- @referenced_nodes[node.name] ||= node
- end
- return nodes.values.collect {|node| node.domain.name}
- end
-
- #
- # Generates entries needed for updating /etc/hosts on a node (as a hash).
- #
- # Argument `nodes` can be nil or a list of nodes. If nil, only include the
- # IPs of the other nodes this @node as has encountered (plus all mx nodes).
- #
- # Also, for virtual machines, we use the local address if this @node is in
- # the same location as the node in question.
- #
- # We include the ssh public key for each host, so that the hash can also
- # be used to generate the /etc/ssh/known_hosts
- #
- def hosts_file(nodes=nil)
- if nodes.nil?
- if @referenced_nodes && @referenced_nodes.any?
- nodes = @referenced_nodes
- nodes = nodes.merge(nodes_like_me[:services => 'mx']) # all nodes always need to communicate with mx nodes.
- end
- end
- return {} unless nodes
- hosts = {}
- my_location = @node['location'] ? @node['location']['name'] : nil
- nodes.each_node do |node|
- hosts[node.name] = {
- 'ip_address' => node.ip_address,
- 'domain_internal' => node.domain.internal,
- 'domain_full' => node.domain.full,
- 'port' => node.ssh.port
- }
- node_location = node['location'] ? node['location']['name'] : nil
- if my_location == node_location
- if facts = @node.manager.facts[node.name]
- if facts['ec2_public_ipv4']
- hosts[node.name]['ip_address'] = facts['ec2_public_ipv4']
- end
- end
- end
- host_pub_key = Util::read_file([:node_ssh_pub_key,node.name])
- if host_pub_key
- hosts[node.name]['host_pub_key'] = host_pub_key
- end
- end
- hosts
- end
-
- end
-end \ No newline at end of file
diff --git a/provider_base/lib/macros/keys.rb b/provider_base/lib/macros/keys.rb
deleted file mode 100644
index 0ed7ccd0..00000000
--- a/provider_base/lib/macros/keys.rb
+++ /dev/null
@@ -1,83 +0,0 @@
-# encoding: utf-8
-
-#
-# Macro for dealing with cryptographic keys
-#
-
-module LeapCli
- module Macro
-
- #
- # return the path to the tor public key
- # generating key if it is missing
- #
- def tor_public_key_path(path_name, key_type)
- path = file_path(path_name)
- if path.nil?
- generate_tor_key(key_type)
- file_path(path_name)
- else
- path
- end
- end
-
- #
- # return the path to the tor private key
- # generating key if it is missing
- #
- def tor_private_key_path(path_name, key_type)
- path = file_path(path_name)
- if path.nil?
- generate_tor_key(key_type)
- file_path(path_name)
- else
- path
- end
- end
-
- #
- # Generates a onion_address from a public RSA key file.
- #
- # path_name is the named path of the Tor public key.
- #
- # Basically, an onion address is nothing more than a base32 encoding
- # of the first 10 bytes of a sha1 digest of the public key.
- #
- # Additionally, Tor ignores the 22 byte header of the public key
- # before taking the sha1 digest.
- #
- def onion_address(path_name)
- require 'base32'
- require 'base64'
- require 'openssl'
- path = Path.find_file([path_name, self.name])
- if path && File.exists?(path)
- public_key_str = File.readlines(path).grep(/^[^-]/).join
- public_key = Base64.decode64(public_key_str)
- public_key = public_key.slice(22..-1) # Tor ignores the 22 byte SPKI header
- sha1sum = Digest::SHA1.new.digest(public_key)
- Base32.encode(sha1sum.slice(0,10)).downcase
- else
- LeapCli.log :warning, 'Tor public key file "%s" does not exist' % tor_public_key_path
- end
- end
-
- private
-
- def generate_tor_key(key_type)
- if key_type == 'RSA'
- require 'certificate_authority'
- keypair = CertificateAuthority::MemoryKeyMaterial.new
- bit_size = 1024
- LeapCli.log :generating, "%s bit RSA Tor key" % bit_size do
- keypair.generate_key(bit_size)
- LeapCli::Util.write_file! [:node_tor_priv_key, self.name], keypair.private_key.to_pem
- LeapCli::Util.write_file! [:node_tor_pub_key, self.name], keypair.public_key.to_pem
- end
- else
- LeapCli.bail! 'tor.key.type of %s is not yet supported' % key_type
- end
- end
-
- end
-end
diff --git a/provider_base/lib/macros/nodes.rb b/provider_base/lib/macros/nodes.rb
deleted file mode 100644
index 8b961cbc..00000000
--- a/provider_base/lib/macros/nodes.rb
+++ /dev/null
@@ -1,88 +0,0 @@
-# encoding: utf-8
-
-##
-## node related macros
-##
-
-module LeapCli
- module Macro
-
- #
- # the list of all the nodes
- #
- def nodes
- global.nodes
- end
-
- #
- # simple alias for global.provider
- #
- def provider
- global.provider
- end
-
- #
- # returns a list of nodes that match the same environment
- #
- # if @node.environment is not set, we return other nodes
- # where environment is not set.
- #
- def nodes_like_me
- nodes[:environment => @node.environment]
- end
-
- #
- # returns a list of nodes that match the location name
- # and environment of @node.
- #
- def nodes_near_me
- if @node['location'] && @node['location']['name']
- nodes_like_me['location.name' => @node.location.name]
- else
- nodes_like_me['location' => nil]
- end
- end
-
- #
- #
- # picks a node out from the node list in such a way that:
- #
- # (1) which nodes picked which nodes is saved in secrets.json
- # (2) when other nodes call this macro with the same node list, they are guaranteed to get a different node
- # (3) if all the nodes in the pick_node list have been picked, remaining nodes are distributed randomly.
- #
- # if the node_list is empty, an exception is raised.
- # if node_list size is 1, then that node is returned and nothing is
- # memorized via the secrets.json file.
- #
- # `label` is needed to distinguish between pools of nodes for different purposes.
- #
- # TODO: more evenly balance after all the nodes have been picked.
- #
- def pick_node(label, node_list)
- if node_list.any?
- if node_list.size == 1
- return node_list.values.first
- else
- secrets_key = "pick_node(:#{label},#{node_list.keys.sort.join(',')})"
- secrets_value = @manager.secrets.retrieve(secrets_key, @node.environment) || {}
- secrets_value[@node.name] ||= begin
- node_to_pick = nil
- node_list.each_node do |node|
- next if secrets_value.values.include?(node.name)
- node_to_pick = node.name
- end
- node_to_pick ||= secrets_value.values.shuffle.first # all picked already, so pick a random one.
- node_to_pick
- end
- picked_node_name = secrets_value[@node.name]
- @manager.secrets.set(secrets_key, secrets_value, @node.environment)
- return node_list[picked_node_name]
- end
- else
- raise ArgumentError.new('pick_node(node_list): node_list cannot be empty')
- end
- end
-
- end
-end \ No newline at end of file
diff --git a/provider_base/lib/macros/secrets.rb b/provider_base/lib/macros/secrets.rb
deleted file mode 100644
index 8d1feb55..00000000
--- a/provider_base/lib/macros/secrets.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# encoding: utf-8
-
-require 'base32'
-
-module LeapCli
- module Macro
-
- #
- # inserts a named secret, generating it if needed.
- #
- # manager.export_secrets should be called later to capture any newly generated secrets.
- #
- # +length+ is the character length of the generated password.
- #
- def secret(name, length=32)
- manager.secrets.set(name, @node.environment) { Util::Secret.generate(length) }
- end
-
- # inserts a base32 encoded secret
- def base32_secret(name, length=20)
- manager.secrets.set(name, @node.environment) { Base32.encode(Util::Secret.generate(length)) }
- end
-
- # Picks a random obfsproxy port from given range
- def rand_range(name, range)
- manager.secrets.set(name, @node.environment) { rand(range) }
- end
-
- #
- # inserts an hexidecimal secret string, generating it if needed.
- #
- # +bit_length+ is the bits in the secret, (ie length of resulting hex string will be bit_length/4)
- #
- def hex_secret(name, bit_length=128)
- manager.secrets.set(name, @node.environment) { Util::Secret.generate_hex(bit_length) }
- end
-
- end
-end \ No newline at end of file
diff --git a/provider_base/lib/macros/stunnel.rb b/provider_base/lib/macros/stunnel.rb
deleted file mode 100644
index f16308c7..00000000
--- a/provider_base/lib/macros/stunnel.rb
+++ /dev/null
@@ -1,95 +0,0 @@
-##
-## STUNNEL
-##
-
-#
-# About stunnel
-# --------------------------
-#
-# The network looks like this:
-#
-# From the client's perspective:
-#
-# |------- stunnel client --------------| |---------- stunnel server -----------------------|
-# consumer app -> localhost:accept_port -> connect:connect_port -> ??
-#
-# From the server's perspective:
-#
-# |------- stunnel client --------------| |---------- stunnel server -----------------------|
-# ?? -> *:accept_port -> localhost:connect_port -> service
-#
-
-module LeapCli
- module Macro
-
- #
- # stunnel configuration for the client side.
- #
- # +node_list+ is a ObjectList of nodes running stunnel servers.
- #
- # +port+ is the real port of the ultimate service running on the servers
- # that the client wants to connect to.
- #
- # * accept_port is the port on localhost to which local clients
- # can connect. it is auto generated serially.
- #
- # * connect_port is the port on the stunnel server to connect to.
- # it is auto generated from the +port+ argument.
- #
- # generates an entry appropriate to be passed directly to
- # create_resources(stunnel::service, hiera('..'), defaults)
- #
- # local ports are automatically generated, starting at 4000
- # and incrementing in sorted order (by node name).
- #
- def stunnel_client(node_list, port, options={})
- @next_stunnel_port ||= 4000
- node_list = listify(node_list)
- hostnames(node_list) # record the hosts
- result = Config::ObjectList.new
- node_list.each_node do |node|
- if node.name != self.name || options[:include_self]
- result["#{node.name}_#{port}"] = Config::Object[
- 'accept_port', @next_stunnel_port,
- 'connect', node.domain.internal,
- 'connect_port', stunnel_port(port),
- 'original_port', port
- ]
- @next_stunnel_port += 1
- end
- end
- result
- end
-
- #
- # generates a stunnel server entry.
- #
- # +port+ is the real port targeted service.
- #
- # * `accept_port` is the publicly bound port
- # * `connect_port` is the port that the local service is running on.
- #
- def stunnel_server(port)
- {
- "accept_port" => stunnel_port(port),
- "connect_port" => port
- }
- end
-
- private
-
- #
- # maps a real port to a stunnel port (used as the connect_port in the client config
- # and the accept_port in the server config)
- #
- def stunnel_port(port)
- port = port.to_i
- if port < 50000
- return port + 10000
- else
- return port - 10000
- end
- end
-
- end
-end \ No newline at end of file