Merge branch 'develop'

10 files changed, 396 insertions, 0 deletions

diff --git a/puppet/modules/squid_deb_proxy b/puppet/modules/squid_deb_proxy
deleted file mode 160000
-Subproject e796aac43aa9781069e167459253d040504c209
diff --git a/puppet/modules/squid_deb_proxy/.gitrepo b/puppet/modules/squid_deb_proxy/.gitrepo
new file mode 100644
index 00000000..78765952
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/.gitrepo
@@ -0,0 +1,11 @@
+; DO NOT EDIT (unless you know what you are doing)
+;
+; This subdirectory is a git "subrepo", and this file is maintained by the
+; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
+;
+[subrepo]
+	remote = https://leap.se/git/puppet_squid_deb_proxy
+	branch = master
+	commit = 08bfaccaea01fd2d334946428504e71a51748e3d
+	parent = a658f5c30ada5e03468257f90d08f6cd2ba25488
+	cmdver = 0.3.0
diff --git a/puppet/modules/squid_deb_proxy/README.md b/puppet/modules/squid_deb_proxy/README.md
new file mode 100644
index 00000000..c183c826
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/README.md
@@ -0,0 +1,8 @@
+This module installes squid-deb-proxy (server or client)
+see https://launchpad.net/squid-deb-proxy for more details
+
+Debian Support
+==============
+
+* As of 2013-07, squid-deb-proxy just arrived in jessie, so you need to
+  configure apt to use jessie. 
diff --git a/puppet/modules/squid_deb_proxy/files/Debian/squid-deb-proxy.conf b/puppet/modules/squid_deb_proxy/files/Debian/squid-deb-proxy.conf
new file mode 100644
index 00000000..2a528f84
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/files/Debian/squid-deb-proxy.conf
@@ -0,0 +1,91 @@
+
+#       WELCOME TO SQUID DEB PROXY
+#       ------------------
+#
+#       This config file is a version of a squid proxy file optimized
+#	as a configuration for a caching proxy for Debian/Ubuntu systems.
+#
+#       More information about squid and its configuration can be found here
+#       http://www.squid-cache.org/ and in the FAQ
+
+# settings that you may want to customize
+# ---------------------------------------
+
+# this file contains private networks (10.0.0.0/8, 172.16.0.0/12,
+# 192.168.0.0/16) by default, you can add/remove additional allowed
+# source networks in it to customize it for your setup
+acl allowed_networks src "/etc/squid-deb-proxy/autogenerated/allowed-networks-src.acl"
+
+# this file contains the archive mirrors by default,
+# if you use a different mirror, add it there
+acl to_archive_mirrors dstdomain "/etc/squid-deb-proxy/autogenerated/mirror-dstdomain.acl"
+
+# this contains the package blacklist
+acl blockedpkgs urlpath_regex "/etc/squid-deb-proxy/autogenerated/pkg-blacklist-regexp.acl"
+
+# default to a different port than stock squid
+http_port 8000
+
+# -------------------------------------------------
+# settings below probably do not need customization
+
+# user visible name
+visible_hostname squid-deb-proxy
+
+# we need a big cache, some debs are huge
+maximum_object_size 512 MB
+
+# use a different dir than stock squid and default to 40G
+cache_dir aufs /var/cache/squid-deb-proxy 40000 16 256
+
+# use different logs
+cache_access_log /var/log/squid-deb-proxy/access.log
+cache_log /var/log/squid-deb-proxy/cache.log
+cache_store_log /var/log/squid-deb-proxy/store.log
+
+# tweaks to speed things up
+cache_mem 200 MB
+maximum_object_size_in_memory 10240 KB
+
+# pid
+pid_filename /var/run/squid-deb-proxy.pid
+
+# refresh pattern for debs and udebs
+refresh_pattern deb$   129600 100% 129600
+refresh_pattern udeb$   129600 100% 129600
+refresh_pattern tar.gz$  129600 100% 129600
+
+# always refresh Packages and Release files
+refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz)$ 0 0% 0
+refresh_pattern \/Release(|\.gpg)$ 0 0% 0
+refresh_pattern \/InRelease$ 0 0% 0
+
+# handle meta-release and changelogs.ubuntu.com special
+# (fine to have this on debian too)
+refresh_pattern changelogs.ubuntu.com/*  0  1% 1
+
+# only allow connects to ports for http, https
+acl Safe_ports port 80
+acl Safe_ports port 443 563   
+
+# only allow ports we trust
+http_access deny !Safe_ports
+
+# do not allow to download from the pkg blacklist
+http_access deny blockedpkgs
+
+# allow access only to official archive mirrors
+# uncomment the third and fouth line to permit any unlisted domain
+http_access deny !to_archive_mirrors
+#http_access allow !to_archive_mirrors
+
+# don't cache domains not listed in the mirrors file
+# uncomment the third and fourth line to cache any unlisted domains
+cache deny !to_archive_mirrors
+#cache allow !to_archive_mirrors
+
+# allow access from our network and localhost
+http_access allow allowed_networks
+
+# And finally deny all other access to this proxy
+http_access deny all
diff --git a/puppet/modules/squid_deb_proxy/files/Ubuntu/squid-deb-proxy.conf b/puppet/modules/squid_deb_proxy/files/Ubuntu/squid-deb-proxy.conf
new file mode 100644
index 00000000..ab5bac8a
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/files/Ubuntu/squid-deb-proxy.conf
@@ -0,0 +1,89 @@
+
+#       WELCOME TO SQUID DEB PROXY
+#       ------------------
+#
+#       This config file is a version of a squid proxy file optimized
+#	as a configuration for a caching proxy for Ubuntu systems.
+#
+#       More information about squid and its configuration can be found here
+#       http://www.squid-cache.org/ and in the FAQ
+
+# settings that you may want to customize
+# ---------------------------------------
+
+# this file contains private networks (10.0.0.0/8, 172.16.0.0/12,
+# 192.168.0.0/16) by default, you can add/remove additional allowed
+# source networks in it to customize it for your setup
+acl allowed_networks src "/etc/squid-deb-proxy/autogenerated/allowed-networks-src.acl"
+
+# this file contains the *archive.ubuntu.com mirrors by default,
+# if you use a different mirror, add it there
+acl to_ubuntu_mirrors dstdomain "/etc/squid-deb-proxy/autogenerated/mirror-dstdomain.acl"
+
+# this contains the package blacklist
+acl blockedpkgs urlpath_regex "/etc/squid-deb-proxy/autogenerated/pkg-blacklist-regexp.acl"
+
+# default to a different port than stock squid
+http_port 8000
+
+# -------------------------------------------------
+# settings below probably do not need customization
+
+# user visible name
+visible_hostname squid-deb-proxy
+
+# we need a big cache, some debs are huge
+maximum_object_size 512 MB
+
+# use a different dir than stock squid and default to 40G
+cache_dir aufs /var/cache/squid-deb-proxy 40000 16 256
+
+# use different logs
+cache_access_log /var/log/squid-deb-proxy/access.log
+cache_log /var/log/squid-deb-proxy/cache.log
+cache_store_log /var/log/squid-deb-proxy/store.log
+
+# tweaks to speed things up
+cache_mem 200 MB
+maximum_object_size_in_memory 10240 KB
+
+# pid
+pid_filename /var/run/squid-deb-proxy.pid
+
+# refresh pattern for debs and udebs
+refresh_pattern deb$   129600 100% 129600
+refresh_pattern udeb$   129600 100% 129600
+refresh_pattern tar.gz$  129600 100% 129600
+
+# always refresh Packages and Release files
+refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz)$ 0 0% 0
+refresh_pattern \/Release(|\.gpg)$ 0 0% 0
+
+# handle meta-release and changelogs.ubuntu.com special
+refresh_pattern changelogs.ubuntu.com/*  0  1% 1
+
+# only allow connects to ports for http, https
+acl Safe_ports port 80
+acl Safe_ports port 443 563   
+
+# only allow ports we trust
+http_access deny !Safe_ports
+
+# do not allow to download from the pkg blacklist
+http_access deny blockedpkgs
+
+# allow access only to official ubuntu mirrors
+# uncomment the third and fouth line to permit any unlisted domain
+http_access deny !to_ubuntu_mirrors
+#http_access allow !to_ubuntu_mirrors
+
+# don't cache domains not listed in the mirrors file
+# uncomment the third and fourth line to cache any unlisted domains
+cache deny !to_ubuntu_mirrors
+#cache allow !to_ubuntu_mirrors
+
+# allow access from our network and localhost
+http_access allow allowed_networks
+
+# And finally deny all other access to this proxy
+http_access deny all
diff --git a/puppet/modules/squid_deb_proxy/files/allowed-networks-src.acl.d/20-custom b/puppet/modules/squid_deb_proxy/files/allowed-networks-src.acl.d/20-custom
new file mode 100644
index 00000000..d4058b80
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/files/allowed-networks-src.acl.d/20-custom
@@ -0,0 +1 @@
+# managed by puppet
diff --git a/puppet/modules/squid_deb_proxy/files/client/apt-avahi-discover b/puppet/modules/squid_deb_proxy/files/client/apt-avahi-discover
new file mode 100755
index 00000000..8dbc1be2
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/files/client/apt-avahi-discover
@@ -0,0 +1,138 @@
+#!/usr/bin/python
+#
+# use avahi to find a _apt_proxy._tcp provider and return
+# a http proxy string suitable for apt
+
+import asyncore
+import functools
+import os
+import socket
+import sys
+import time
+from subprocess	import Popen, PIPE, call
+from syslog import syslog, LOG_INFO, LOG_USER
+
+DEFAULT_CONNECT_TIMEOUT_SEC = 2
+
+def DEBUG(msg):
+    if "--debug" in sys.argv:
+        sys.stderr.write(msg + "\n")
+
+
+def get_avahi_discover_timeout():
+    APT_AVAHI_TIMEOUT_VAR = "APT::Avahi-Discover::Timeout"
+    p = Popen(
+        ["/usr/bin/apt-config", "shell", "TIMEOUT", APT_AVAHI_TIMEOUT_VAR], 
+        stdout=PIPE)
+    stdout, stderr = p.communicate()
+    if not stdout:
+        DEBUG(
+            "no timeout set, using default '%s'" % DEFAULT_CONNECT_TIMEOUT_SEC)
+        return DEFAULT_CONNECT_TIMEOUT_SEC
+    if not stdout.startswith("TIMEOUT="):
+        raise ValueError("got unexpected apt-config output: '%s'" % stdout)
+    varname, sep, value = stdout.strip().partition("=")
+    timeout = int(value.strip("'"))
+    DEBUG("using timeout: '%s'" % timeout)
+    return timeout
+
+@functools.total_ordering
+class AptAvahiClient(asyncore.dispatcher):
+    def __init__(self, addr):
+        asyncore.dispatcher.__init__(self)
+        if is_ipv6(addr[0]):
+            self.create_socket(socket.AF_INET6, socket.SOCK_STREAM)
+            self.connect( (addr[0], addr[1], 0, 0) )
+        else:
+            self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+            self.connect(addr)
+        self._time_init = time.time()
+        self.time_to_connect = sys.maxint
+        self.address = addr
+    def handle_connect(self):
+        self.time_to_connect = time.time() - self._time_init
+        self.close()
+    def __eq__(self, other):
+        return self.time_to_connect == other.time_to_connect
+    def __lt__(self, other):
+        return self.time_to_connect < other.time_to_connect
+    def __repr__(self):
+        return "<%s> %s: %s" % (
+            self.__class__.__name__, self.addr, self.time_to_connect)
+    def log(self, message):
+        syslog((LOG_INFO|LOG_USER), '%s\n' % str(message))
+    def log_info(self, message, type='info'):
+        if type not in self.ignore_log_types:
+            self.log('%s: %s' % (type, message))
+
+
+def is_ipv6(a):
+    return ':' in a
+
+def is_linklocal(addr):
+    # Link-local should start with fe80 and six null bytes
+    return addr.startswith("fe80::")
+
+def get_proxy_host_port_from_avahi():
+    service = '_apt_proxy._tcp'
+
+    # Obtain all of the services addresses from avahi, pulling the IPv6
+    # addresses to the top.
+    addr4 = []
+    addr6 = []
+    p = Popen(['avahi-browse', '-kprtf', service], stdout=PIPE)
+    DEBUG("avahi-browse output:")
+    for line in p.stdout:
+        DEBUG(" '%s'" % line)
+        if line.startswith('='):
+            tokens = line.split(';')
+            addr = tokens[7]
+            port = int(tokens[8])
+            if is_ipv6(addr):
+                # We need to skip ipv6 link-local addresses since 
+                # APT can't use them
+                if not is_linklocal(addr):
+                    addr6.append((addr, port))
+            else:
+                addr4.append((addr, port))
+
+    # Run through the offered addresses and see if we we have a bound local
+    # address for it.
+    addrs = []
+    for (ip, port) in addr6 + addr4:
+        try:
+            res = socket.getaddrinfo(ip, port, 0, 0, 0, socket.AI_ADDRCONFIG)
+            if res:
+                addrs.append((ip, port))
+        except socket.gaierror:
+            pass
+    if not addrs:
+        return None
+    
+    # sort by answering speed
+    hosts = []
+    for addr in addrs:
+        hosts.append(AptAvahiClient(addr))
+    # 2s timeout, arbitray
+    timeout = get_avahi_discover_timeout()
+    asyncore.loop(timeout=timeout)
+    DEBUG("sorted hosts: '%s'" % sorted(hosts))
+
+    # No host wanted to connect
+    if (all(h.time_to_connect == sys.maxint for h in hosts)):
+        return None
+
+    fastest_host = sorted(hosts)[0]
+    fastest_address = fastest_host.address
+    return fastest_address
+
+
+if __name__ == "__main__":
+    # Dump the approved address out in an appropriate format.
+    address = get_proxy_host_port_from_avahi()
+    if address:
+        (ip, port) = address
+        if is_ipv6(ip):
+            print "http://[%s]:%s/" % (ip, port)
+        else:
+            print "http://%s:%s/" % (ip, port)
diff --git a/puppet/modules/squid_deb_proxy/files/mirror-dstdomain.acl.d/20-custom b/puppet/modules/squid_deb_proxy/files/mirror-dstdomain.acl.d/20-custom
new file mode 100644
index 00000000..d4058b80
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/files/mirror-dstdomain.acl.d/20-custom
@@ -0,0 +1 @@
+# managed by puppet
diff --git a/puppet/modules/squid_deb_proxy/manifests/client.pp b/puppet/modules/squid_deb_proxy/manifests/client.pp
new file mode 100644
index 00000000..049f740a
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/manifests/client.pp
@@ -0,0 +1,16 @@
+# install squid-deb-proxy-client package
+class squid_deb_proxy::client {
+  package { 'squid-deb-proxy-client':
+    ensure => installed,
+  } ->
+
+  # ship newer client discover script than includes in squid-deb-proxy-client
+  # v. 0.8.13 to fix error messages being sent to stdout instead of stderr,
+  # see https://bugs.launchpad.net/ubuntu/+source/squid-deb-proxy/+bug/1505670
+  file { '/usr/share/squid-deb-proxy-client/apt-avahi-discover':
+    source => 'puppet:///modules/squid_deb_proxy/client/apt-avahi-discover',
+    mode   => '0755',
+    owner  => 'root',
+    group  => 'root',
+  }
+}
diff --git a/puppet/modules/squid_deb_proxy/manifests/server.pp b/puppet/modules/squid_deb_proxy/manifests/server.pp
new file mode 100644
index 00000000..b6c3b2ed
--- /dev/null
+++ b/puppet/modules/squid_deb_proxy/manifests/server.pp
@@ -0,0 +1,41 @@
+class squid_deb_proxy::server {
+  package { 'squid-deb-proxy':
+    ensure => installed,
+  }
+
+  service { 'squid-deb-proxy':
+    ensure    => running,
+    hasstatus => false,
+    require   => Package[ 'squid-deb-proxy' ],
+  }
+
+  file {'/etc/squid-deb-proxy/mirror-dstdomain.acl.d/20-custom':
+    source  => [ 'puppet:///modules/site_squid_deb_proxy/mirror-dstdomain.acl.d/20-custom',
+      'puppet:///modules/squid_deb_proxy/mirror-dstdomain.acl.d/20-custom' ],
+    notify  => Service[ 'squid-deb-proxy' ],
+    require => Package[ 'squid-deb-proxy' ],
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+  }
+
+  file {'/etc/squid-deb-proxy/allowed-networks-src.acl.d/20-custom':
+    source  => [ 'puppet:///modules/site_squid_deb_proxy/allowed-networks-src.acl.d/20-custom',
+      'puppet:///modules/squid_deb_proxy/allowed-networks-src.acl.d/20-custom' ],
+    notify  => Service[ 'squid-deb-proxy' ],
+    require => Package[ 'squid-deb-proxy' ],
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+  }
+
+  file { '/etc/squid-deb-proxy/squid-deb-proxy.conf':
+    source  => [ "puppet:///modules/site_squid_deb_proxy/${::operatingsystem}/squid-deb-proxy.conf",
+      "puppet:///modules/squid_deb_proxy/${::operatingsystem}/squid-deb-proxy.conf" ],
+    notify  => Service[ 'squid-deb-proxy' ],
+    require => Package[ 'squid-deb-proxy' ],
+    mode    => '0644',
+    owner   => 'root',
+    group   => 'root',
+  }
+}