diff options
| author | Micah Anderson <micah@riseup.net> | 2017-11-28 11:35:01 -0500 |
|---|---|---|
| committer | Micah Anderson <micah@riseup.net> | 2017-11-28 11:35:01 -0500 |
| commit | 0d251e2ceddd3e02ed8bba8725830689dbdd1397 (patch) | |
| tree | 37d7096d9e458ca1e6431dff8a2f571553011c44 | |
| parent | 93a181d44e2d8163ae44945aac1b6477e268170d (diff) | |
| parent | bf6c56d86c7ba45e7ca766d990a9e9162025e5ac (diff) | |
Merge tag 'refs/tags/0.10.0' into stable
Release 0.10.0
234 files changed, 2776 insertions, 2055 deletions
@@ -30,3 +30,6 @@ tests/platform-ci/provider/users/gitlab-runner-platform/* /tests/platform-ci/provider/test /builds + +/.ruby-version +/.rbenv-gemsets diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c315b2d2..a7a79d11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ image: 0xacab.org:4567/leap/docker/ruby:latest # persistent on the gitlab-runner so we don't need to install from # scratch on every pipeline cache: - key: "$CI_BUILD_REF_NAME" + key: "global_platform_cache_between_pipelines" untracked: true paths: - tests/platform-ci/vendor/ @@ -51,22 +51,20 @@ catalog: script: - su -c '/usr/local/bin/bundle exec rake catalog' cirunner -# rspec: -# stage: rspec -# script: -# - /usr/local/bin/bundle exec rake spec - -# The deploy_test job is run on any merge request. This is used to ensure that -# the merge request will deploy and test properly. It is not run when the merge -# request is accepted into master, instead the 'latest' job below is run -# instead. -deploy_test: +deploy_test:master: stage: deploy except: - master script: - su -c 'set -o pipefail; stdbuf -oL -eL ./ci-build.sh | ts' cirunner +deploy_test:staging: + stage: deploy + variables: + COMPONENT: "staging" + script: + - su -c 'set -o pipefail; stdbuf -oL -eL ./ci-build.sh | ts' cirunner + # However, sometimes it's important to have a way of triggering a deploy # from scratch manually even from the master branch, when i.e. new packages # got uploaded to the master component of the platform deb repo. @@ -75,14 +73,54 @@ deploy_test:manual: only: - master when: manual + allow_failure: false script: - su -c 'set -o pipefail; stdbuf -oL -eL ./ci-build.sh | ts' cirunner # Test upgrades from the latetest release to latest HEAD upgrade_test: stage: deploy - when: manual script: # Allow unpriviledged user to checkout last release of leap_platform - chown cirunner:cirunner -R ../.. - - su -c 'set -o pipefail; stdbuf -oL -eL ./ci-build.sh | ts' cirunner || sleep 9000 + - su -c 'set -o pipefail; stdbuf -oL -eL ./ci-build.sh | ts' cirunner + +# Latest job will only run on the master branch, which means all merge requests +# that are created from branches don't get to deploy to the latest-ci server. +# When a merge request is merged, then the latest job will deploy the code to +# the latest provider, and the deployment will be recorded in an environment +# named 'latest' +ci.leap.se: + stage: deploy + environment: + name: staging + only: + - master@leap/platform + script: + - > + su -c '/usr/bin/unbuffer /bin/bash -o pipefail ./ci-build.sh | + /usr/bin/ts' cirunner + +demo.bitmask.net: + stage: deploy + environment: + name: production/demo/vpn + only: + - master + when: manual + script: + - > + su -c '/usr/bin/unbuffer /bin/bash -o pipefail ./ci-build.sh | + /usr/bin/ts' cirunner + +mail.bitmask.net: + stage: deploy + environment: + name: production/demo/mail + only: + - master + when: manual + script: + - > + su -c '/usr/bin/unbuffer /bin/bash -o pipefail ./ci-build.sh | + /usr/bin/ts' cirunner diff --git a/.mailmap b/.mailmap deleted file mode 100644 index aee70b0a..00000000 --- a/.mailmap +++ /dev/null @@ -1,8 +0,0 @@ -Varac <varacanero@zeromail.org> -Micah Anderson <micah@leap.se> Micah Anderson <micah@riseup.net> -Micah Anderson <micah@leap.se> micah <micah@leap.se> -Kwadronaut <kwadronaut@leap.se> -Elijah <elijah@riseup.net> elijah <elijah@ChrUbuntu.(none)> -Elijah <elijah@riseup.net> elijah <elijah@riseup.net> -Leap Admins <admin@leap.se> root <root@localhost> - @@ -1,3 +1,133 @@ +Platform 0.10 +------------------------------------------------ + +The main focus for Platform 0.10 was to update of all client-side daemons to +newest releases, like Soledad and OpenVPN. This introduces a *compatibility +change*: by setting the platform version to 0.10, it also requires client 0.9.4 +or later. We also switched the development branch to the 'master' branch and are +creating a branch called 0.10.x to push hot-fixes during the 0.10 life-cycle. + +Note: This will be the last major release of the LEAP Platform for Debian +Jessie. We will continue to support 0.10 with minor releases with important +security and bug fixes, but the next major release will require an upgrade to +Stretch. + +New Features: + +* Tor single-hop onion service capability. +* `leap info` is now run after deploy +* Timestamps are added to deployments +* Missing ssh host keys are generated on node init +* Private networking support for local Vagrant development +* Static sites get lets encrypt support +* add command `leap node disable`, `leap node enable`, `leap ping` + +Notable Changes: + +* Removed haproxy because we don't support multi-node couchdb installations anymore (#8144). +* Disable nagios notification emails (#8772). +* Fix layout of apt repository (#8888) +* Limit what archive signing keys are accepted for the leap debian repository packages (#8425). +* Monitor the Webapp logs for errors (#5174). +* Moved development to the master branch. +* Rewrite leap_cli ssh code +* Debian wheezy was fully deprecated +* Restructure package archives to enable auto packaging, and CI testing +* Significant CI improvements +* Troubleshooting information added to `leap user ls` +* Couchdb service is no longer required on soledad nodes (#8693) +* Tor service refactored (#8864), and v3 hidden service support added (#8879) +* Fixed unattended-upgrades (#8891) +* Alert on 409 responses for webapp +* Many other issues resolved, full list: https://0xacab.org/groups/leap/milestones/platform-010?title=Platform+0.10 + +Upgrading: + +If you have a node with the service 'tor' defined, you will need to change it to +be either 'tor-relay', or 'tor-exit'. Look in your provider directory under the +nodes directory for any .json file that has a 'services' section with 'tor' +defined, change that to the correct tor service you are wanting to deploy. + +Make sure you have the correct version of leap_cli + + workstation$ sudo gem install leap_cli --version=1.9 + +If you are upgrading from a version previous to 0.9, please follow those upgrade +instructions before upgrading to 0.10. + +Prepare your platform source by checking out the 0.10.x branch: + + workstation$ cd leap_platform + workstation$ git fetch + workstation$ git checkout 0.10.x + +Then, deploy: + + workstation$ cd $PROVIDER_DIR + workstation$ leap deploy + workstation$ leap test + +After deployment, if the leap test does not succeed, you should +investigate. Please see below for some post-deployment upgrade steps that you +may need to perform. + +Starting with Soledad Server 0.9.0, the CouchDB database schema was changed to +improve speed of the server side storage backend. If you provided email, you +will need to run the migration script, otherwise it is unnecessary. Until you +migrate, soledad will refuse to start. + +To run the migration script, do the following (replacing $PROVIDER_DIR, +$COUCHDB_NODE, $MX_NODE, and $SOLEDAD_NODE with your values): + +First backup your couchdb databases, just to be safe. NOTE: This can take some +time and will place several hundred megabytes of data into +/var/backups/couchdb. The size and time depends on how many users there are on +your system. For example, 15k users took approximately 25 minutes and 308M of +space: + + workstation$ leap ssh $COUCHDB_NODE + server# cd /srv/leap/couchdb/scripts + server# ./cleanup-user-dbs + server# time ./couchdb_dumpall.sh + + Once that has finished, then its time to run the migration: + + workstation$ cd $PROVIDER_DIR + workstation$ leap run 'systemctl leap_mx stop' $MX_NODE + workstation$ leap run --stream '/usr/share/soledad-server/migration/0.9/migrate.py --log-file /var/log/leap/soledad_migration --verbose --do-migrate' $SOLEDAD_NODE + wait for it to finish (will print DONE) + rerun if interrupted + workstation$ leap deploy + workstation$ leap test + +Known Issues: + +If you have been deploying from our master branch (ie: unstable code), you might +end up with a broken sources line for apt. If you get the following: + WARNING: The following packages cannot be authenticated! + +Then you should remove the files on your nodes inside +/var/lib/puppet/modules/apt/keys and deploy again. (#8862, #8876) + +* When upgrading, sometimes systemd does not report the correct state of a + daemon. The daemon will be not running, but systemd thinks it is. The symptom + of this is that a deploy will succeed but `leap test` will fail. To fix, you + can run `systemctl stop DAEMON` and then `systemctl start DAEMON` on the + affected host (systemctl restart seems to work less reliably). + +Includes: + +* leap_web: 0.9.2 +* nickserver: 0.10.0 +* leap-mx: 0.10.1 +* soledad-server: 0.10.5 + +Commits: https://0xacab.org/groups/leap/milestones/platform-010?title=Platform+0.10 + +For details on about all the changes included in this release please consult the +[LEAP platform 0.10 milestone](https://0xacab.org/leap/platform/milestones/7 ). + + Platform 0.9 -------------------------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..1eb6d3fd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,21 @@ +# Contributed Files + +## Commit Template + +To install our commit template, use following command (use --global to use it in your global .gitconfig): + + git config commit.template "~/path_to_leap_platform/contrib/leap-commit-template" + +To use for all projects: + + git config --global commit.template "~/path_to_leap_platform/contrib/leap-commit-template" + +## Signing commits + +We very much appreciate signed commits, you can stop forgetting it like this: + + git config commit.gpgsign true + +To enable for all projects: + + git config --global commit.gpgsign true @@ -1,7 +1,7 @@ Leap Platform ============================= -[](https://0xacab.org/leap/platform/commits/develop) +[](https://0xacab.org/leap/platform/commits/master) The LEAP Platform is set of complementary packages and server recipes to automate the maintenance of LEAP services in a hardened Debian environment. Its @@ -37,6 +37,13 @@ For a live deployment of the platform, the number of servers that is required depends on your needs and which services you want to deploy. At the moment, the LEAP Platform supports servers with a base Debian Jessie installation. +Upgrading +============================= + +If you are upgrading from a previous version of the LEAP Platform, take special +care to follow the instructions detailed in the CHANGES.md to move from one +release to the next. + Troubleshooting ============================= diff --git a/bin/debug.sh b/bin/debug.sh index d6f37542..4f29f300 100755 --- a/bin/debug.sh +++ b/bin/debug.sh @@ -2,7 +2,7 @@ # debug script to be run on remote servers # called from leap_cli with the 'leap debug' cmd -apps='(leap|pixelated|stunnel|couch|soledad|haproxy)' +apps='(leap|pixelated|stunnel|couch|soledad)' facts='(apt_running |^architecture |^augeasversion |^couchdb_.* |^debian_.* |^dhcp_enabled |^domain |^facterversion |^filesystems |^fqdn |^hardwaremodel |^hostname |^interface.* |^ipaddress.* |^is_pe |^is_virtual |^kernel.* |^lib |^lsb.* |^memory.* |^mtu_.* |^netmask.* |^network_.* |^operatingsystem |^os.* |^path |^physicalprocessorcount |^processor.* |^ps |^puppetversion |^root_home |^rsyslog_version |^rubysitedir |^rubyversion |^selinux |^ssh_version |^swapfree.* |^swapsize.* |^type |^virtual)' @@ -12,6 +12,10 @@ export FACTERLIB="/srv/leap/puppet/modules/apache/lib/facter:/srv/leap/puppet/mo facter 2>/dev/null | egrep -i "$facts" +# show leap debian repo used +echo -e '\n\n' +cat /etc/apt/sources.list.d/leap*.list + # query installed versions echo -e '\n\n' dpkg -l | egrep "$apps" @@ -24,6 +28,3 @@ ps aux|egrep "$apps" echo -e '\n\n' echo -e "Last deploy:\n" tail -2 /var/log/leap/deploy-summary.log - - - diff --git a/bin/node_init b/bin/node_init index b55cfed3..24345b47 100755 --- a/bin/node_init +++ b/bin/node_init @@ -22,10 +22,6 @@ if ! egrep -q "$DEBIAN_VERSION" /etc/debian_version; then exit 1 fi mkdir -p $LEAP_DIR -if ! grep -q -e '^en_US.UTF-8' /etc/locale.gen; then - echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen - /usr/sbin/locale-gen -fi # # UPDATE PACKAGES @@ -83,9 +79,20 @@ if [[ $exit_code -ne 0 ]]; then exit $exit_code fi +# need to have the locales package from above +if ! grep -q -e '^en_US.UTF-8' /etc/locale.gen 2> /dev/null; then + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen + /usr/sbin/locale-gen +fi + # # FINALIZE # mkdir -p $HIERA_DIR chmod 0755 $HIERA_DIR touch $INIT_FILE + +# Sometimes not all keys are already generated, happens more often +# with VMs +# that would give us errors in the get_ssh_keys_cmd during node init +/usr/bin/ssh-keygen -A diff --git a/bin/run_tests b/bin/run_tests index 8450a9bf..241a3fa2 100755 --- a/bin/run_tests +++ b/bin/run_tests @@ -507,7 +507,7 @@ end if ARGV.include?('--debug') || ARGV.include?('-d') DEBUG=true - require 'debugger' + require 'byebug' else DEBUG=false end diff --git a/contrib/README.md b/contrib/README.md deleted file mode 100644 index e836bc7e..00000000 --- a/contrib/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Contributed Files - -## Commit Template - -to install this commit template, use following cmd (use --global to use it in your global .gitconfig): - - git config [--global] commit.template "~/path_to_leap_platform/contrib/commit-template.txt" - - diff --git a/contrib/commit-template.txt b/contrib/commit-template.txt deleted file mode 100644 index 9a1fa81b..00000000 --- a/contrib/commit-template.txt +++ /dev/null @@ -1,7 +0,0 @@ -#[bug|feat|docs|style|refactor|test|pkg|i18n] - -#- Tested: [local singlenode|local multinode|citest|unstable.bitmask.net] -#- Resolves: #XYZ -#- Related: #XYZ -#- Documentation: #XYZ -#- Releases: XYZ diff --git a/contrib/leap-commit-template b/contrib/leap-commit-template new file mode 100644 index 00000000..73cc6436 --- /dev/null +++ b/contrib/leap-commit-template @@ -0,0 +1,20 @@ +# Bug|Feat|CI|Docs|Style|Tests|I18n|Vagrant|Lint: [style: https://github.com/m1foley/fit-commit] + + +# [Story about your commit.] + +# Use one or more of these things: +# - Fixes|Closes #3210 +# - Related: #1234 + +#example: +# +# Bug: fix apache systemd insanity +# +# Apache2 systemd autorestart was failing because of a gremlin that was entering +# through a wormhole. This fixes #985843 by closing the wormhole so no further +# gremlins can come through. +# +# # Use one of these things: +# [- Fixes|Closes #5234] +# [- Related: #1234] diff --git a/docs/en/guide/keys-and-certificates.html b/docs/en/guide/keys-and-certificates.html index f5f83066..95c08cb9 100644 --- a/docs/en/guide/keys-and-certificates.html +++ b/docs/en/guide/keys-and-certificates.html @@ -181,6 +181,25 @@ Keys and Certificates - LEAP Platform Documentation <li> <a href="keys-and-certificates/index.html#renewing-a-certificate">Renewing a certificate</a> </li> + <li> + <a href="keys-and-certificates/index.html#issues">Issues</a> + <ol> + <li> + <a href="keys-and-certificates/index.html#certs-already-expired">Certs already expired</a> + <ol> + <li> + <a href="keys-and-certificates/index.html#install-the-official-acme-client">Install the official acme client</a> + </li> + <li> + <a href="keys-and-certificates/index.html#fetch-cert">Fetch cert</a> + </li> + <li> + <a href="keys-and-certificates/index.html#deploy-the-certs">Deploy the certs</a> + </li> + </ol> + </li> + </ol> + </li> </ol> </li> </ol></div> @@ -445,6 +464,76 @@ workstation$ leap deploy <p>There is no need to create a new CSR: renewing will reuse the old private key and the old CSR. It is especially important to not create a new CSR if you have advertised public key pins using HPKP.</p> +<h2><a name="issues"></a>Issues</h2> + +<h3><a name="certs-already-expired"></a>Certs already expired</h3> + +<p>When a cert is already expired, you can get into a possible deadlock situation on your servers which you can only resolve manually at the moment.</p> + +<h4><a name="install-the-official-acme-client"></a>Install the official acme client</h4> + +<p>Log in to your webapp node and install the <code>certbot</code> package:</p> + +<pre><code>server$ apt install -t jessie-backports certbot +</code></pre> + +<h4><a name="fetch-cert"></a>Fetch cert</h4> + +<p>Stop apache so the letsencrypt client can bind to port 80:</p> + +<pre><code>server$ systemctl stop apache2 +</code></pre> + +<p>Fetch the certs</p> + +<pre><code>server$ certbot certonly --standalone --email admin@$(hostname -d) -d $(hostname -d) -d api.$(hostname -d) -d $(hostname -f) -d nicknym.$(hostname -d) +</code></pre> + +<p>This will put the certs and keys into <code>/etc/letsencrypt/live/DOMAIN/</code>.</p> + +<p>Now, go to your workstation’s provider configuration directory and copy the newly created files from the server to your local config. You will override existing files so please make a backup before proceeding, or use a version control system to track changes.</p> + +<pre><code>workstation$ cd PATH_TO_PROVIDER_CONFIG +</code></pre> + +<p>Copy the Certificate</p> + +<pre><code>workstation$ scp 'root@SERVER:/etc/letsencrypt/live/$(hostname -d)/cert.pem' files/cert/DOMAIN.crt +</code></pre> + +<p>Copy the private key</p> + +<pre><code>workstation$ scp 'root@SERVER:/etc/letsencrypt/live/$(hostname -d)/privkey.pem' files/cert/DOMAIN.key +</code></pre> + +<p>Copy the CA chain cert</p> + +<pre><code>workstation$ scp 'root@SERVER:/etc/letsencrypt/live/$(hostname -d)/fullchain.pem' files/cert/commercial_ca.crt +</code></pre> + +<h4><a name="deploy-the-certs"></a>Deploy the certs</h4> + +<p>Now you only need to deploy the certs</p> + +<pre><code>workstation$ leap deploy +</code></pre> + +<p>This will put them into the right locations which are:</p> + +<ul> +<li><code>/etc/x509/certs/leap_commercial.crt</code> for the certificate</li> +<li><code>/etc/x509/./keys/leap_commercial.key</code> for the private key</li> +<li><code>/usr/local/share/ca-certificates/leap_commercial_ca.crt</code> for the CA chain cert.</li> +</ul> + + +<p>Start apache2 again</p> + +<pre><code>server$ systemctl start apache2 +</code></pre> + +<p>Done! In the future please make sure to always renew letsencrypt certificates before they expire ;).</p> + </div> </div> </body> diff --git a/docs/en/guide/keys-and-certificates/index.html b/docs/en/guide/keys-and-certificates/index.html index 016a03a7..95279270 100644 --- a/docs/en/guide/keys-and-certificates/index.html +++ b/docs/en/guide/keys-and-certificates/index.html @@ -181,6 +181,25 @@ Keys and Certificates - LEAP Platform Documentation <li> <a href="index.html#renewing-a-certificate">Renewing a certificate</a> </li> + <li> + <a href="index.html#issues">Issues</a> + <ol> + <li> + <a href="index.html#certs-already-expired">Certs already expired</a> + <ol> + <li> + <a href="index.html#install-the-official-acme-client">Install the official acme client</a> + </li> + <li> + <a href="index.html#fetch-cert">Fetch cert</a> + </li> + <li> + <a href="index.html#deploy-the-certs">Deploy the certs</a> + </li> + </ol> + </li> + </ol> + </li> </ol> </li> </ol></div> @@ -445,6 +464,76 @@ workstation$ leap deploy <p>There is no need to create a new CSR: renewing will reuse the old private key and the old CSR. It is especially important to not create a new CSR if you have advertised public key pins using HPKP.</p> +<h2><a name="issues"></a>Issues</h2> + +<h3><a name="certs-already-expired"></a>Certs already expired</h3> + +<p>When a cert is already expired, you can get into a possible deadlock situation on your servers which you can only resolve manually at the moment.</p> + +<h4><a name="install-the-official-acme-client"></a>Install the official acme client</h4> + +<p>Log in to your webapp node and install the <code>certbot</code> package:</p> + +<pre><code>server$ apt install -t jessie-backports certbot +</code></pre> + +<h4><a name="fetch-cert"></a>Fetch cert</h4> + +<p>Stop apache so the letsencrypt client can bind to port 80:</p> + +<pre><code>server$ systemctl stop apache2 +</code></pre> + +<p>Fetch the certs</p> + +<pre><code>server$ certbot certonly --standalone --email admin@$(hostname -d) -d $(hostname -d) -d api.$(hostname -d) -d $(hostname -f) -d nicknym.$(hostname -d) +</code></pre> + +<p>This will put the certs and keys into <code>/etc/letsencrypt/live/DOMAIN/</code>.</p> + +<p>Now, go to your workstation’s provider configuration directory and copy the newly created files from the server to your local config. You will override existing files so please make a backup before proceeding, or use a version control system to track changes.</p> + +<pre><code>workstation$ cd PATH_TO_PROVIDER_CONFIG +</code></pre> + +<p>Copy the Certificate</p> + +<pre><code>workstation$ scp 'root@SERVER:/etc/letsencrypt/live/$(hostname -d)/cert.pem' files/cert/DOMAIN.crt +</code></pre> + +<p>Copy the private key</p> + +<pre><code>workstation$ scp 'root@SERVER:/etc/letsencrypt/live/$(hostname -d)/privkey.pem' files/cert/DOMAIN.key +</code></pre> + +<p>Copy the CA chain cert</p> + +<pre><code>workstation$ scp 'root@SERVER:/etc/letsencrypt/live/$(hostname -d)/fullchain.pem' files/cert/commercial_ca.crt +</code></pre> + +<h4><a name="deploy-the-certs"></a>Deploy the certs</h4> + +<p>Now you only need to deploy the certs</p> + +<pre><code>workstation$ leap deploy +</code></pre> + +<p>This will put them into the right locations which are:</p> + +<ul> +<li><code>/etc/x509/certs/leap_commercial.crt</code> for the certificate</li> +<li><code>/etc/x509/./keys/leap_commercial.key</code> for the private key</li> +<li><code>/usr/local/share/ca-certificates/leap_commercial_ca.crt</code> for the CA chain cert.</li> +</ul> + + +<p>Start apache2 again</p> + +<pre><code>server$ systemctl start apache2 +</code></pre> + +<p>Done! In the future please make sure to always renew letsencrypt certificates before they expire ;).</p> + </div> </div> </body> diff --git a/docs/en/guide/virtual-machines.html b/docs/en/guide/virtual-machines.html index 5cee9a40..28be3211 100644 --- a/docs/en/guide/virtual-machines.html +++ b/docs/en/guide/virtual-machines.html @@ -142,7 +142,7 @@ Virtual Machines - LEAP Platform Documentation <h2><a name="introduction"></a>Introduction</h2> -<p>You can use the <code>leap</code> command line to easily remote virtual machines.</p> +<p>You can use the <code>leap</code> command line to easily manage remote virtual machines.</p> <p>Note: there are two types of virtual machines that <code>leap</code> can handle:</p> @@ -220,6 +220,7 @@ Virtual Machines - LEAP Platform Documentation <ul> <li><a href="https://aws.amazon.com/ec2/instance-types/">Available instance types for AWS</a></li> +<li><a href="https://aws.amazon.com/pt/blogs/security/wheres-my-secret-access-key/">Where’s My Secret Access Key?</a></li> </ul> @@ -245,6 +246,11 @@ leap vm start mynode <pre><code>leap vm add mynode services:webapp tags:seattle vm.options.InstanceType:t2.small </code></pre> +<p>For an email provider installation, you should specify the following seeds:</p> + +<pre><code>leap vm add mynode services:webapp,couchdb,soledad,mx +</code></pre> + <p>Check to see what the status is of all VMs:</p> <pre><code>leap vm status diff --git a/docs/en/guide/virtual-machines/index.html b/docs/en/guide/virtual-machines/index.html index da0da107..20d45a77 100644 --- a/docs/en/guide/virtual-machines/index.html +++ b/docs/en/guide/virtual-machines/index.html @@ -142,7 +142,7 @@ Virtual Machines - LEAP Platform Documentation <h2><a name="introduction"></a>Introduction</h2> -<p>You can use the <code>leap</code> command line to easily remote virtual machines.</p> +<p>You can use the <code>leap</code> command line to easily manage remote virtual machines.</p> <p>Note: there are two types of virtual machines that <code>leap</code> can handle:</p> @@ -220,6 +220,7 @@ Virtual Machines - LEAP Platform Documentation <ul> <li><a href="https://aws.amazon.com/ec2/instance-types/">Available instance types for AWS</a></li> +<li><a href="https://aws.amazon.com/pt/blogs/security/wheres-my-secret-access-key/">Where’s My Secret Access Key?</a></li> </ul> @@ -245,6 +246,11 @@ leap vm start mynode <pre><code>leap vm add mynode services:webapp tags:seattle vm.options.InstanceType:t2.small </code></pre> +<p>For an email provider installation, you should specify the following seeds:</p> + +<pre><code>leap vm add mynode services:webapp,couchdb,soledad,mx +</code></pre> + <p>Check to see what the status is of all VMs:</p> <pre><code>leap vm status diff --git a/docs/en/services.html b/docs/en/services.html index 55211e64..8237b3e4 100644 --- a/docs/en/services.html +++ b/docs/en/services.html @@ -235,7 +235,7 @@ Services - LEAP Platform Documentation <h2> <a href='services/tor.html'>tor</a> </h2> -<div class='summary'>Tor exit node or hidden service</div> +<div class='summary'>Tor services: relay, exit node and hidden service</div> </div> <div class=' page-summary'> <h2> diff --git a/docs/en/services/couchdb.html b/docs/en/services/couchdb.html index 6de6455c..43f7cfac 100644 --- a/docs/en/services/couchdb.html +++ b/docs/en/services/couchdb.html @@ -215,7 +215,7 @@ couchdb - LEAP Platform Documentation <ul> <li>search for the “user_id” field</li> -<li>in this example <a href="mailto:testuser@example.org">testuser@example.org</a> uses the database user-665e004870ee17aa4c94331ff3cd59eb</li> +<li>in this example <a href="mailto:testuser@example.org">testuser@example.org</a> uses the database user-665e004870ee17aa4c94331ff3cd59eb</li> </ul> diff --git a/docs/en/services/couchdb/index.html b/docs/en/services/couchdb/index.html index 10043db6..b48c4eb7 100644 --- a/docs/en/services/couchdb/index.html +++ b/docs/en/services/couchdb/index.html @@ -215,7 +215,7 @@ couchdb - LEAP Platform Documentation <ul> <li>search for the “user_id” field</li> -<li>in this example <a href="mailto:testuser@example.org">testuser@example.org</a> uses the database user-665e004870ee17aa4c94331ff3cd59eb</li> +<li>in this example <a href="mailto:testuser@example.org">testuser@example.org</a> uses the database user-665e004870ee17aa4c94331ff3cd59eb</li> </ul> diff --git a/docs/en/services/index.html b/docs/en/services/index.html index 6d5c68e1..261cd11b 100644 --- a/docs/en/services/index.html +++ b/docs/en/services/index.html @@ -235,7 +235,7 @@ Services - LEAP Platform Documentation <h2> <a href='tor.html'>tor</a> </h2> -<div class='summary'>Tor exit node or hidden service</div> +<div class='summary'>Tor services: relay, exit node and hidden service</div> </div> <div class=' page-summary'> <h2> diff --git a/docs/en/services/mx.html b/docs/en/services/mx.html index 8e08cfe0..aa41186a 100644 --- a/docs/en/services/mx.html +++ b/docs/en/services/mx.html @@ -156,8 +156,8 @@ mx - LEAP Platform Documentation <ol> <li>alias lists: by specifying an array of destination addresses, as in the case of “flock”, the single email will get copied to each address.</li> -<li>chained resolution: alias resolution will recursively continue until there are no more matching aliases. For example, “flock” is resolved to “robin”, which then gets resolved to “<a href="mailto:robin@bird.org">robin@bird.org</a>”.</li> -<li>virtual domains: by specifying the full domain, as in the case of “<a href="mailto:chickadee@avian.org">chickadee@avian.org</a>”, the alias will work for any domain you want. Of course, the MX record for that domain must point to appropriate MX servers, but otherwise you don’t need to do any additional configuration.</li> +<li>chained resolution: alias resolution will recursively continue until there are no more matching aliases. For example, “flock” is resolved to “robin”, which then gets resolved to “<a href="mailto:robin@bird.org">robin@bird.org</a>”.</li> +<li>virtual domains: by specifying the full domain, as in the case of “<a href="mailto:chickadee@avian.org">chickadee@avian.org</a>”, the alias will work for any domain you want. Of course, the MX record for that domain must point to appropriate MX servers, but otherwise you don’t need to do any additional configuration.</li> <li>local delivery: for testing purposes, it is often useful to copy all incoming mail for a particular address and send those copies to another address. You can do this by adding “@deliver.local” as one of the destination addresses. When “@local.delivery” is found, alias resolution stops and the mail is delivered to that username.</li> </ol> diff --git a/docs/en/services/mx/index.html b/docs/en/services/mx/index.html index 6899e0cc..048f5198 100644 --- a/docs/en/services/mx/index.html +++ b/docs/en/services/mx/index.html @@ -156,8 +156,8 @@ mx - LEAP Platform Documentation <ol> <li>alias lists: by specifying an array of destination addresses, as in the case of “flock”, the single email will get copied to each address.</li> -<li>chained resolution: alias resolution will recursively continue until there are no more matching aliases. For example, “flock” is resolved to “robin”, which then gets resolved to “<a href="mailto:robin@bird.org">robin@bird.org</a>”.</li> -<li>virtual domains: by specifying the full domain, as in the case of “<a href="mailto:chickadee@avian.org">chickadee@avian.org</a>”, the alias will work for any domain you want. Of course, the MX record for that domain must point to appropriate MX servers, but otherwise you don’t need to do any additional configuration.</li> +<li>chained resolution: alias resolution will recursively continue until there are no more matching aliases. For example, “flock” is resolved to “robin”, which then gets resolved to “<a href="mailto:robin@bird.org">robin@bird.org</a>”.</li> +<li>virtual domains: by specifying the full domain, as in the case of “<a href="mailto:chickadee@avian.org">chickadee@avian.org</a>”, the alias will work for any domain you want. Of course, the MX record for that domain must point to appropriate MX servers, but otherwise you don’t need to do any additional configuration.</li> <li>local delivery: for testing purposes, it is often useful to copy all incoming mail for a particular address and send those copies to another address. You can do this by adding “@deliver.local” as one of the destination addresses. When “@local.delivery” is found, alias resolution stops and the mail is delivered to that username.</li> </ol> diff --git a/docs/en/services/openvpn.html b/docs/en/services/openvpn.html index e5fe1128..1a420e21 100644 --- a/docs/en/services/openvpn.html +++ b/docs/en/services/openvpn.html @@ -133,8 +133,8 @@ openvpn - LEAP Platform Documentation <p><em>Essential configuration</em></p> <ul> -<li><code>openvpn.gateway_address</code>: The address that OpenVPN daemon is bound to and that VPN clients connect to.</li> <li><code>ip_address</code>: The main IP of the server, and the egress address for outgoing traffic.</li> +<li><code>openvpn.gateway_address</code>: A secondary address on the same machine (sharing the same interface, or on a separate interface). The OpenVPN daemon is bound to this address and VPN clients connect to it.</li> </ul> diff --git a/docs/en/services/openvpn/index.html b/docs/en/services/openvpn/index.html index 4a9dc993..23866436 100644 --- a/docs/en/services/openvpn/index.html +++ b/docs/en/services/openvpn/index.html @@ -133,8 +133,8 @@ openvpn - LEAP Platform Documentation <p><em>Essential configuration</em></p> <ul> -<li><code>openvpn.gateway_address</code>: The address that OpenVPN daemon is bound to and that VPN clients connect to.</li> <li><code>ip_address</code>: The main IP of the server, and the egress address for outgoing traffic.</li> +<li><code>openvpn.gateway_address</code>: A secondary address on the same machine (sharing the same interface, or on a separate interface). The OpenVPN daemon is bound to this address and VPN clients connect to it.</li> </ul> diff --git a/docs/en/services/tor.html b/docs/en/services/tor.html index f649c086..1f6ce112 100644 --- a/docs/en/services/tor.html +++ b/docs/en/services/tor.html @@ -110,7 +110,7 @@ tor - LEAP Platform Documentation <div id='title-box'> <h1>tor</h1> -<div id='summary'>Tor exit node or hidden service</div> +<div id='summary'>Tor services: relay, exit node and hidden service</div> </div> <div id='content-box'> <div id="TOC"><ol> @@ -124,33 +124,53 @@ tor - LEAP Platform Documentation <h2><a name="topology"></a>Topology</h2> -<p>Nodes with <code>tor</code> service will run a Tor exit or hidden service, depending on what other service it is paired with:</p> +<p>Nodes with <code>tor</code> service will run a Tor relay with some pre-defined settings, which can be changed with some configuration (see <em>Configuration</em> below). You can enable an exit or a hidden service with additional configuration.</p> + +<h2><a name="configuration"></a>Configuration</h2> + +<p>By default, if a node has service ‘tor’ configured, it will run a tor relay (not an exit). The relay will be configured with bandwidth limitations, contacts, a nickname and a family. The defaults for these (shown below), can be overridden as desired.</p> <ul> -<li><code>tor</code> + <code>openvpn</code>: when combined with <code>openvpn</code> nodes, <code>tor</code> will create a Tor exit node to provide extra cover traffic for the VPN. This can be especially useful if there are VPN gateways without much traffic.</li> -<li><code>tor</code> + <code>webapp</code>: when combined with a <code>webapp</code> node, the <code>tor</code> service will make the webapp and the API available via .onion hidden service.</li> -<li><code>tor</code> stand alone: a regular Tor exit node.</li> +<li><code>tor.bandwidth_rate</code>: the max bandwidth allocated to Tor, in KB per second, when used as an exit node (default: 6550 KB/sec).</li> +<li><code>tor.type</code>: what type of tor node to make, at this moment only ‘exit’ is supported. If not specified, acts as a relay.</li> +<li><code>tor.contacts</code>: the contact information for the relay (default: the list of provider contacts)</li> +<li><code>tor.nickname</code>: the nickname of the relay (default: a combination of the node name and a hash of the family)</li> +<li><code>tor.family</code>: a list of the other nicknames that are part of the same provider</li> +<li><code>tor.hidden_service</code>: to enable a hidden service, set ‘active’ to be true (see below for an example), do <em>not</em> configure “services”: [“tor”] for the node!</li> </ul> -<p>If activated, you can list the hidden service .onion addresses this way:</p> +<p>Examples:</p> -<p> leap ls –print tor.hidden_service.address tor</p> +<p>To add a relay to a node:</p> -<p>Then just add ‘.onion’ to the end of the printed addresses.</p> +<pre><code>{ + "services": ["tor"] +} +</code></pre> -<h2><a name="configuration"></a>Configuration</h2> +<p>To enable a hidden service, without a relay, do <em>not</em> specify the tor service (it is not considered secure to have a node configured as a relay and a hidden service at the same time, see: <a href="https://trac.torproject.org/8742">https://trac.torproject.org/8742</a>), instead configure the node to have the following:</p> -<ul> -<li><code>tor.bandwidth_rate</code>: the max bandwidth allocated to Tor, in KB per second, when used as an exit node.</li> -</ul> +<pre><code>{ + "tor": { + "hidden_service": { + "active": true + } +} +</code></pre> +<p>If activated, you can list the hidden service .onion addresses this way:</p> + +<p> leap ls –print tor.hidden_service.address tor</p> + +<p>Then just add ‘.onion’ to the end of the printed addresses.</p> -<p>For example:</p> +<p>To enable a Tor exit node:</p> <pre><code>{ "tor": { - "bandwidth_rate": 6550 + "bandwidth_rate": 6550, + "type": "exit" } } </code></pre> diff --git a/docs/en/services/tor/index.html b/docs/en/services/tor/index.html index 8fecf152..a6380d90 100644 --- a/docs/en/services/tor/index.html +++ b/docs/en/services/tor/index.html @@ -110,7 +110,7 @@ tor - LEAP Platform Documentation <div id='title-box'> <h1>tor</h1> -<div id='summary'>Tor exit node or hidden service</div> +<div id='summary'>Tor services: relay, exit node and hidden service</div> </div> <div id='content-box'> <div id="TOC"><ol> @@ -124,33 +124,53 @@ tor - LEAP Platform Documentation <h2><a name="topology"></a>Topology</h2> -<p>Nodes with <code>tor</code> service will run a Tor exit or hidden service, depending on what other service it is paired with:</p> +<p>Nodes with <code>tor</code> service will run a Tor relay with some pre-defined settings, which can be changed with some configuration (see <em>Configuration</em> below). You can enable an exit or a hidden service with additional configuration.</p> + +<h2><a name="configuration"></a>Configuration</h2> + +<p>By default, if a node has service ‘tor’ configured, it will run a tor relay (not an exit). The relay will be configured with bandwidth limitations, contacts, a nickname and a family. The defaults for these (shown below), can be overridden as desired.</p> <ul> -<li><code>tor</code> + <code>openvpn</code>: when combined with <code>openvpn</code> nodes, <code>tor</code> will create a Tor exit node to provide extra cover traffic for the VPN. This can be especially useful if there are VPN gateways without much traffic.</li> -<li><code>tor</code> + <code>webapp</code>: when combined with a <code>webapp</code> node, the <code>tor</code> service will make the webapp and the API available via .onion hidden service.</li> -<li><code>tor</code> stand alone: a regular Tor exit node.</li> +<li><code>tor.bandwidth_rate</code>: the max bandwidth allocated to Tor, in KB per second, when used as an exit node (default: 6550 KB/sec).</li> +<li><code>tor.type</code>: what type of tor node to make, at this moment only ‘exit’ is supported. If not specified, acts as a relay.</li> +<li><code>tor.contacts</code>: the contact information for the relay (default: the list of provider contacts)</li> +<li><code>tor.nickname</code>: the nickname of the relay (default: a combination of the node name and a hash of the family)</li> +<li><code>tor.family</code>: a list of the other nicknames that are part of the same provider</li> +<li><code>tor.hidden_service</code>: to enable a hidden service, set ‘active’ to be true (see below for an example), do <em>not</em> configure “services”: [“tor”] for the node!</li> </ul> -<p>If activated, you can list the hidden service .onion addresses this way:</p> +<p>Examples:</p> -<p> leap ls –print tor.hidden_service.address tor</p> +<p>To add a relay to a node:</p> -<p>Then just add ‘.onion’ to the end of the printed addresses.</p> +<pre><code>{ + "services": ["tor"] +} +</code></pre> -<h2><a name="configuration"></a>Configuration</h2> +<p>To enable a hidden service, without a relay, do <em>not</em> specify the tor service (it is not considered secure to have a node configured as a relay and a hidden service at the same time, see: <a href="https://trac.torproject.org/8742">https://trac.torproject.org/8742</a>), instead configure the node to have the following:</p> -<ul> -<li><code>tor.bandwidth_rate</code>: the max bandwidth allocated to Tor, in KB per second, when used as an exit node.</li> -</ul> +<pre><code>{ + "tor": { + "hidden_service": { + "active": true + } +} +</code></pre> +<p>If activated, you can list the hidden service .onion addresses this way:</p> + +<p> leap ls –print tor.hidden_service.address tor</p> + +<p>Then just add ‘.onion’ to the end of the printed addresses.</p> -<p>For example:</p> +<p>To enable a Tor exit node:</p> <pre><code>{ "tor": { - "bandwidth_rate": 6550 + "bandwidth_rate": 6550, + "type": "exit" } } </code></pre> diff --git a/docs/en/troubleshooting/known-issues.html b/docs/en/troubleshooting/known-issues.html index 607970b1..72c64f96 100644 --- a/docs/en/troubleshooting/known-issues.html +++ b/docs/en/troubleshooting/known-issues.html @@ -232,6 +232,8 @@ users/userx/otherkey_ssh.pub <p>It is not possible to actually use the EIP openvpn server on vagrant nodes (see: <a href="https://leap.se/code/issues/2401">https://leap.se/code/issues/2401</a>)</p> +<p>Proxmox virtualization isn’t supported because it wants to overwrite our resolv.conf (see: <a href="https://leap.se/code/issues/8683">https://leap.se/code/issues/8683</a>)</p> + </div> </div> </body> diff --git a/docs/en/troubleshooting/known-issues/index.html b/docs/en/troubleshooting/known-issues/index.html index eee3b120..3d3d05a8 100644 --- a/docs/en/troubleshooting/known-issues/index.html +++ b/docs/en/troubleshooting/known-issues/index.html @@ -232,6 +232,8 @@ users/userx/otherkey_ssh.pub <p>It is not possible to actually use the EIP openvpn server on vagrant nodes (see: <a href="https://leap.se/code/issues/2401">https://leap.se/code/issues/2401</a>)</p> +<p>Proxmox virtualization isn’t supported because it wants to overwrite our resolv.conf (see: <a href="https://leap.se/code/issues/8683">https://leap.se/code/issues/8683</a>)</p> + </div> </div> </body> diff --git a/docs/en/troubleshooting/where-to-look.html b/docs/en/troubleshooting/where-to-look.html index a1207aca..93cbbe97 100644 --- a/docs/en/troubleshooting/where-to-look.html +++ b/docs/en/troubleshooting/where-to-look.html @@ -115,9 +115,6 @@ Where to look - LEAP Platform Documentation <a href="where-to-look/index.html#places-to-look-for-errors">Places to look for errors</a> </li> <li> - <a href="where-to-look/index.html#is-haproxy-ok">Is haproxy ok ?</a> - </li> - <li> <a href="where-to-look/index.html#is-couchdb-accessible-through-stunnel">Is couchdb accessible through stunnel ?</a> </li> <li> @@ -216,22 +213,10 @@ Where to look - LEAP Platform Documentation </ul> -<h2><a name="is-haproxy-ok"></a>Is haproxy ok ?</h2> - -<pre><code>curl -s -X GET "http://127.0.0.1:4096" -</code></pre> - <h2><a name="is-couchdb-accessible-through-stunnel"></a>Is couchdb accessible through stunnel ?</h2> -<ul> -<li><p>Depending on how many couch nodes you have, increase the port for every test -(see /etc/haproxy/haproxy.cfg for the server/port mapping):</p> - -<p> curl -s -X GET “<a href="http://127.0.0.1:4000">http://127.0.0.1:4000</a>” - curl -s -X GET “<a href="http://127.0.0.1:4001">http://127.0.0.1:4001</a>” - …</p></li> -</ul> - +<pre><code>curl -s -X GET "http://127.0.0.1:4000" +</code></pre> <h2><a name="check-couchdb-acl-as-admin"></a>Check couchdb acl as admin</h2> @@ -240,8 +225,8 @@ cat /srv/leap/webapp/config/couchdb.yml.admin # see username and password echo "machine 127.0.0.1 login admin password <PASSWORD>" > /etc/couchdb/couchdb-admin.netrc chmod 600 /etc/couchdb/couchdb-admin.netrc -curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096" -curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096/_all_dbs" +curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4000" +curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4000/_all_dbs" </code></pre> <h2><a name="check-couchdb-acl-as-unpriviledged-user"></a>Check couchdb acl as unpriviledged user</h2> @@ -250,8 +235,8 @@ curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4 echo "machine 127.0.0.1 login webapp password <PASSWORD>" > /etc/couchdb/couchdb-webapp.netrc chmod 600 /etc/couchdb/couchdb-webapp.netrc -curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096" -curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096/_all_dbs" +curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4000" +curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4000/_all_dbs" </code></pre> <h2><a name="all-urls-accessible"></a>All URLs accessible ?</h2> @@ -350,15 +335,8 @@ curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1: <h2><a name="is-couchdb-accessible-through-stunnel-2"></a>Is couchdb accessible through stunnel ?</h2> -<ul> -<li><p>Depending on how many couch nodes you have, increase the port for every test -(see /etc/haproxy/haproxy.cfg for the server/port mapping):</p> - -<p> curl -s -X GET “<a href="http://127.0.0.1:4000">http://127.0.0.1:4000</a>” - curl -s -X GET “<a href="http://127.0.0.1:4001">http://127.0.0.1:4001</a>” - …</p></li> -</ul> - +<pre><code>curl -s -X GET "http://127.0.0.1:4000" +</code></pre> <h2><a name="query-leap-mx"></a>Query leap-mx</h2> @@ -398,15 +376,10 @@ curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1: echo "machine 127.0.0.1 login leap_mx password <PASSWORD>" > /etc/couchdb/couchdb-leap_mx.netrc chmod 600 /etc/couchdb/couchdb-leap_mx.netrc -curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/_all_dbs" # pick one "user-<hash>" db -curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/user-de9c77a3d7efbc779c6c20da88e8fb9c" +curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4000/_all_dbs" # pick one "user-<hash>" db +curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4000/user-de9c77a3d7efbc779c6c20da88e8fb9c" </code></pre> -<ul> -<li>you may check multiple times, cause 127.0.0.1:4096 is haproxy load-balancing the different couchdb nodes</li> -</ul> - - <h2><a name="mailspool"></a>Mailspool</h2> <ul> diff --git a/docs/en/troubleshooting/where-to-look/index.html b/docs/en/troubleshooting/where-to-look/index.html index ab3115af..63d05e45 100644 --- a/docs/en/troubleshooting/where-to-look/index.html +++ b/docs/en/troubleshooting/where-to-look/index.html @@ -115,9 +115,6 @@ Where to look - LEAP Platform Documentation <a href="index.html#places-to-look-for-errors">Places to look for errors</a> </li> <li> - <a href="index.html#is-haproxy-ok">Is haproxy ok ?</a> - </li> - <li> <a href="index.html#is-couchdb-accessible-through-stunnel">Is couchdb accessible through stunnel ?</a> </li> <li> @@ -216,22 +213,10 @@ Where to look - LEAP Platform Documentation </ul> -<h2><a name="is-haproxy-ok"></a>Is haproxy ok ?</h2> - -<pre><code>curl -s -X GET "http://127.0.0.1:4096" -</code></pre> - <h2><a name="is-couchdb-accessible-through-stunnel"></a>Is couchdb accessible through stunnel ?</h2> -<ul> -<li><p>Depending on how many couch nodes you have, increase the port for every test -(see /etc/haproxy/haproxy.cfg for the server/port mapping):</p> - -<p> curl -s -X GET “<a href="http://127.0.0.1:4000">http://127.0.0.1:4000</a>” - curl -s -X GET “<a href="http://127.0.0.1:4001">http://127.0.0.1:4001</a>” - …</p></li> -</ul> - +<pre><code>curl -s -X GET "http://127.0.0.1:4000" +</code></pre> <h2><a name="check-couchdb-acl-as-admin"></a>Check couchdb acl as admin</h2> @@ -240,8 +225,8 @@ cat /srv/leap/webapp/config/couchdb.yml.admin # see username and password echo "machine 127.0.0.1 login admin password <PASSWORD>" > /etc/couchdb/couchdb-admin.netrc chmod 600 /etc/couchdb/couchdb-admin.netrc -curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096" -curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4096/_all_dbs" +curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4000" +curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4000/_all_dbs" </code></pre> <h2><a name="check-couchdb-acl-as-unpriviledged-user"></a>Check couchdb acl as unpriviledged user</h2> @@ -250,8 +235,8 @@ curl -s --netrc-file /etc/couchdb/couchdb-admin.netrc -X GET "http://127.0.0.1:4 echo "machine 127.0.0.1 login webapp password <PASSWORD>" > /etc/couchdb/couchdb-webapp.netrc chmod 600 /etc/couchdb/couchdb-webapp.netrc -curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096" -curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4096/_all_dbs" +curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4000" +curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1:4000/_all_dbs" </code></pre> <h2><a name="all-urls-accessible"></a>All URLs accessible ?</h2> @@ -350,15 +335,8 @@ curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1: <h2><a name="is-couchdb-accessible-through-stunnel-2"></a>Is couchdb accessible through stunnel ?</h2> -<ul> -<li><p>Depending on how many couch nodes you have, increase the port for every test -(see /etc/haproxy/haproxy.cfg for the server/port mapping):</p> - -<p> curl -s -X GET “<a href="http://127.0.0.1:4000">http://127.0.0.1:4000</a>” - curl -s -X GET “<a href="http://127.0.0.1:4001">http://127.0.0.1:4001</a>” - …</p></li> -</ul> - +<pre><code>curl -s -X GET "http://127.0.0.1:4000" +</code></pre> <h2><a name="query-leap-mx"></a>Query leap-mx</h2> @@ -398,15 +376,10 @@ curl -s --netrc-file /etc/couchdb/couchdb-webapp.netrc -X GET "http://127.0.0.1: echo "machine 127.0.0.1 login leap_mx password <PASSWORD>" > /etc/couchdb/couchdb-leap_mx.netrc chmod 600 /etc/couchdb/couchdb-leap_mx.netrc -curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/_all_dbs" # pick one "user-<hash>" db -curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4096/user-de9c77a3d7efbc779c6c20da88e8fb9c" +curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4000/_all_dbs" # pick one "user-<hash>" db +curl -s --netrc-file /etc/couchdb/couchdb-leap_mx.netrc -X GET "http://127.0.0.1:4000/user-de9c77a3d7efbc779c6c20da88e8fb9c" </code></pre> -<ul> -<li>you may check multiple times, cause 127.0.0.1:4096 is haproxy load-balancing the different couchdb nodes</li> -</ul> - - <h2><a name="mailspool"></a>Mailspool</h2> <ul> diff --git a/docs/en/tutorials/quick-start.html b/docs/en/tutorials/quick-start.html index d2670b30..d275a321 100644 --- a/docs/en/tutorials/quick-start.html +++ b/docs/en/tutorials/quick-start.html @@ -123,6 +123,9 @@ Quick Start Tutorial - LEAP Platform Documentation <a href="quick-start/index.html#install-pre-requisites">Install pre-requisites</a> </li> <li> + <a href="quick-start/index.html#the-platform-recipes">The platform recipes</a> + </li> + <li> <a href="quick-start/index.html#install-the-leap-command-line-utility">Install the LEAP command-line utility</a> </li> </ol> @@ -139,6 +142,9 @@ Quick Start Tutorial - LEAP Platform Documentation <li> <a href="quick-start/index.html#option-b-add-a-local-node">Option B: Add a local node</a> </li> + <li> + <a href="quick-start/index.html#option-c-add-a-virtual-machine-in-the-cloud">Option C: Add a virtual machine in the cloud</a> + </li> </ol> </li> <li> @@ -197,7 +203,7 @@ Quick Start Tutorial - LEAP Platform Documentation <ol> <li>A local Vagrant virtual machine: a Vagrant machine can only be useful for testing.</li> -<li>A real or paravirtualized server: The server must have Debian Jessie installed, and you must be able to SSH into the machine as root. Paravirtualization includes KVM, Xen, OpenStack, Amazon, but not VirtualBox or OpenVZ.</li> +<li>A real or paravirtualized server: The server must have Debian Jessie installed, and you must be able to SSH into the machine as root. Paravirtualization includes KVM, Xen, OpenStack, Amazon, but not VirtualBox or OpenVZ. Proxmox has an known issue <a href="https://leap.se/code/issues/8683">when changing the resolver</a></li> </ol> </li> </ol> @@ -214,15 +220,20 @@ Quick Start Tutorial - LEAP Platform Documentation <h1><a name="prepare-your-workstation"></a>Prepare your workstation</h1> -<p>In order to be able to manage your servers, you need to install the <code>leap</code> command on your workstation:</p> +<p>In order to be able to manage your servers, you need to setup the LEAP Platform on your desktop. This consists of three parts: the platform recipes, the <code>leap</code> command, and your provider instance. We will go over these step-by-step below, you can find more details in the <a href="quick-start/platform.html">platform introduction</a>.</p> <h3><a name="install-pre-requisites"></a>Install pre-requisites</h3> <p>Install core prerequisites on your workstation.</p> -<p><em>Debian & Ubuntu</em></p> +<p><em>Debian Unstable (sid)</em></p> + +<pre><code>workstation$ sudo apt-get install git rsync openssh-client openssl zlib1g-dev +</code></pre> + +<p><em>Other Debian & Ubuntu</em></p> -<pre><code>workstation$ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 +<pre><code>workstation$ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 zlib1g-dev </code></pre> <p><em>Mac OS</em></p> @@ -231,9 +242,38 @@ Quick Start Tutorial - LEAP Platform Documentation workstation$ ruby-install ruby </code></pre> +<h3><a name="the-platform-recipes"></a>The platform recipes</h3> + +<p>The LEAP platform recipes are a set modules designed to work together to provide you everything you need to manage your provider. You typically do not need to modify these, but do need them available for deploying your provider.</p> + +<p>To obtain the platform recipes, simply clone the git repository, and then check out the most recent stable release branch:</p> + +<pre><code>workstation$ git clone -b version/0.9.x https://leap.se/git/leap_platform +</code></pre> + +<p>If you want to get the latest development branch (Beware: it could be unstable !) you could simply use the master branch instead by:</p> + +<pre><code>workstation$ git clone https://leap.se/git/leap_platform +</code></pre> + <h3><a name="install-the-leap-command-line-utility"></a>Install the LEAP command-line utility</h3> -<p>Install the <code>leap</code> command system-wide:</p> +<p>The <code>leap</code> <a href="quick-start/guide/commands.html">command line tool</a> is what you use to manage everything about your provider.</p> + +<p>Keep these rules in mind:</p> + +<ul> +<li><code>leap</code> is run on your workstation: The <code>leap</code> command is always run locally on your workstation, never on a server you are deploying to.</li> +<li><code>leap</code> is run from within a provider instance: The <code>leap</code> command requires that the current working directory is a valid provider instance, except when running <code>leap new</code> to create a new provider instance.</li> +</ul> + + +<p>If on Debian Unstable (sid), simply do this:</p> + +<pre><code>workstation$ sudo apt install leap-cli +</code></pre> + +<p>Otherwise, you will need to do this:</p> <pre><code>workstation$ sudo gem install leap_cli </code></pre> @@ -340,14 +380,18 @@ workstation$ leap cert csr <h3><a name="option-b-add-a-local-node"></a>Option B: Add a local node</h3> -<p>Create a node, with the services “webapp” and “couchdb”, and then start the local virtual machine:</p> +<p>Create a node, with the services “webapp”, “soledad” and “couchdb”, and then start the local virtual machine:</p> -<pre><code>workstation$ leap node add --local wildebeest services:webapp,couchdb +<pre><code>workstation$ leap node add --local wildebeest services:webapp,couchdb,soledad workstation$ leap local start wildebeest </code></pre> <p>It will take a while to download the Virtualbox base box and create the virtual machine.</p> +<h3><a name="option-c-add-a-virtual-machine-in-the-cloud"></a>Option C: Add a virtual machine in the cloud</h3> + +<p>In order to create a provider using the cloud, please follow this <a href="https://leap.se/en/docs/platform/guide/virtual-machines">instructions</a>.</p> + <h1><a name="deploy-your-provider"></a>Deploy your provider</h1> <h3><a name="initialize-the-node"></a>Initialize the node</h3> diff --git a/docs/en/commands.html b/docs/en/tutorials/quick-start/guide/commands.html index e69de29b..e69de29b 100644 --- a/docs/en/commands.html +++ b/docs/en/tutorials/quick-start/guide/commands.html diff --git a/docs/en/tutorials/quick-start/index.html b/docs/en/tutorials/quick-start/index.html index 27b21238..ae617e1b 100644 --- a/docs/en/tutorials/quick-start/index.html +++ b/docs/en/tutorials/quick-start/index.html @@ -123,6 +123,9 @@ Quick Start Tutorial - LEAP Platform Documentation <a href="index.html#install-pre-requisites">Install pre-requisites</a> </li> <li> + <a href="index.html#the-platform-recipes">The platform recipes</a> + </li> + <li> <a href="index.html#install-the-leap-command-line-utility">Install the LEAP command-line utility</a> </li> </ol> @@ -139,6 +142,9 @@ Quick Start Tutorial - LEAP Platform Documentation <li> <a href="index.html#option-b-add-a-local-node">Option B: Add a local node</a> </li> + <li> + <a href="index.html#option-c-add-a-virtual-machine-in-the-cloud">Option C: Add a virtual machine in the cloud</a> + </li> </ol> </li> <li> @@ -197,7 +203,7 @@ Quick Start Tutorial - LEAP Platform Documentation <ol> <li>A local Vagrant virtual machine: a Vagrant machine can only be useful for testing.</li> -<li>A real or paravirtualized server: The server must have Debian Jessie installed, and you must be able to SSH into the machine as root. Paravirtualization includes KVM, Xen, OpenStack, Amazon, but not VirtualBox or OpenVZ.</li> +<li>A real or paravirtualized server: The server must have Debian Jessie installed, and you must be able to SSH into the machine as root. Paravirtualization includes KVM, Xen, OpenStack, Amazon, but not VirtualBox or OpenVZ. Proxmox has an known issue <a href="https://leap.se/code/issues/8683">when changing the resolver</a></li> </ol> </li> </ol> @@ -214,15 +220,20 @@ Quick Start Tutorial - LEAP Platform Documentation <h1><a name="prepare-your-workstation"></a>Prepare your workstation</h1> -<p>In order to be able to manage your servers, you need to install the <code>leap</code> command on your workstation:</p> +<p>In order to be able to manage your servers, you need to setup the LEAP Platform on your desktop. This consists of three parts: the platform recipes, the <code>leap</code> command, and your provider instance. We will go over these step-by-step below, you can find more details in the <a href="platform.html">platform introduction</a>.</p> <h3><a name="install-pre-requisites"></a>Install pre-requisites</h3> <p>Install core prerequisites on your workstation.</p> -<p><em>Debian & Ubuntu</em></p> +<p><em>Debian Unstable (sid)</em></p> + +<pre><code>workstation$ sudo apt-get install git rsync openssh-client openssl zlib1g-dev +</code></pre> + +<p><em>Other Debian & Ubuntu</em></p> -<pre><code>workstation$ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 +<pre><code>workstation$ sudo apt-get install git ruby ruby-dev rsync openssh-client openssl rake make bzip2 zlib1g-dev </code></pre> <p><em>Mac OS</em></p> @@ -231,9 +242,38 @@ Quick Start Tutorial - LEAP Platform Documentation workstation$ ruby-install ruby </code></pre> +<h3><a name="the-platform-recipes"></a>The platform recipes</h3> + +<p>The LEAP platform recipes are a set modules designed to work together to provide you everything you need to manage your provider. You typically do not need to modify these, but do need them available for deploying your provider.</p> + +<p>To obtain the platform recipes, simply clone the git repository, and then check out the most recent stable release branch:</p> + +<pre><code>workstation$ git clone -b version/0.9.x https://leap.se/git/leap_platform +</code></pre> + +<p>If you want to get the latest development branch (Beware: it could be unstable !) you could simply use the master branch instead by:</p> + +<pre><code>workstation$ git clone https://leap.se/git/leap_platform +</code></pre> + <h3><a name="install-the-leap-command-line-utility"></a>Install the LEAP command-line utility</h3> -<p>Install the <code>leap</code> command system-wide:</p> +<p>The <code>leap</code> <a href="guide/commands.html">command line tool</a> is what you use to manage everything about your provider.</p> + +<p>Keep these rules in mind:</p> + +<ul> +<li><code>leap</code> is run on your workstation: The <code>leap</code> command is always run locally on your workstation, never on a server you are deploying to.</li> +<li><code>leap</code> is run from within a provider instance: The <code>leap</code> command requires that the current working directory is a valid provider instance, except when running <code>leap new</code> to create a new provider instance.</li> +</ul> + + +<p>If on Debian Unstable (sid), simply do this:</p> + +<pre><code>workstation$ sudo apt install leap-cli +</code></pre> + +<p>Otherwise, you will need to do this:</p> <pre><code>workstation$ sudo gem install leap_cli </code></pre> @@ -340,14 +380,18 @@ workstation$ leap cert csr <h3><a name="option-b-add-a-local-node"></a>Option B: Add a local node</h3> -<p>Create a node, with the services “webapp” and “couchdb”, and then start the local virtual machine:</p> +<p>Create a node, with the services “webapp”, “soledad” and “couchdb”, and then start the local virtual machine:</p> -<pre><code>workstation$ leap node add --local wildebeest services:webapp,couchdb +<pre><code>workstation$ leap node add --local wildebeest services:webapp,couchdb,soledad workstation$ leap local start wildebeest </code></pre> <p>It will take a while to download the Virtualbox base box and create the virtual machine.</p> +<h3><a name="option-c-add-a-virtual-machine-in-the-cloud"></a>Option C: Add a virtual machine in the cloud</h3> + +<p>In order to create a provider using the cloud, please follow this <a href="https://leap.se/en/docs/platform/guide/virtual-machines">instructions</a>.</p> + <h1><a name="deploy-your-provider"></a>Deploy your provider</h1> <h3><a name="initialize-the-node"></a>Initialize the node</h3> diff --git a/docs/en/tutorials/quick-start/platform.html b/docs/en/tutorials/quick-start/platform.html new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/docs/en/tutorials/quick-start/platform.html diff --git a/docs/en/tutorials/single-node-email.html b/docs/en/tutorials/single-node-email.html index 6678fec3..d3372f91 100644 --- a/docs/en/tutorials/single-node-email.html +++ b/docs/en/tutorials/single-node-email.html @@ -144,11 +144,13 @@ Quick email - LEAP Platform Documentation <p>In our example, we would edit <code>nodes/wildebeest.json</code>:</p> <pre><code>{ - "ip_address": "1.1.1.1", + "ip_address": "XXX.XXX.XXX.XXX", "services": ["couchdb", "webapp", "mx", "soledad"] } </code></pre> +<p>Where “XXX.XXX.XXX.XXX” should be replaced by your IP provider.</p> + <p>Here, we added <code>mx</code> and <code>soledad</code> to the node’s <code>services</code> list. Briefly:</p> <ul> diff --git a/docs/en/tutorials/single-node-email/index.html b/docs/en/tutorials/single-node-email/index.html index 45a1264f..fd501790 100644 --- a/docs/en/tutorials/single-node-email/index.html +++ b/docs/en/tutorials/single-node-email/index.html @@ -144,11 +144,13 @@ Quick email - LEAP Platform Documentation <p>In our example, we would edit <code>nodes/wildebeest.json</code>:</p> <pre><code>{ - "ip_address": "1.1.1.1", + "ip_address": "XXX.XXX.XXX.XXX", "services": ["couchdb", "webapp", "mx", "soledad"] } </code></pre> +<p>Where “XXX.XXX.XXX.XXX” should be replaced by your IP provider.</p> + <p>Here, we added <code>mx</code> and <code>soledad</code> to the node’s <code>services</code> list. Briefly:</p> <ul> diff --git a/docs/en/tutorials/vagrant.html b/docs/en/tutorials/vagrant.html index 3d4f0520..e473ce82 100644 --- a/docs/en/tutorials/vagrant.html +++ b/docs/en/tutorials/vagrant.html @@ -437,12 +437,12 @@ $ leap local save web1 <p>Clone the platform with</p> -<pre><code>git clone --recursive -b develop https://github.com/leapcode/leap_platform.git +<pre><code>git clone https://leap.se/git/leap_platform </code></pre> <p>Start the vagrant box with</p> -<pre><code>cd leap_platform +<pre><code>cd leap_platform/tests/example-provider vagrant up </code></pre> @@ -531,7 +531,7 @@ started by the bitmask client:</p> sudo apt-get install ruby-dev libxslt-dev libxml2-dev libvirt-dev # install the required plugins -vagrant plugin install vagrant-libvirt fog fog-libvirt sahara +vagrant plugin install vagrant-libvirt sahara </code></pre> <p>Log out and then log back in.</p> @@ -585,8 +585,6 @@ virsh pool-autostart vagrant <li><code>Call to virConnectOpen failed: internal error: Unable to locate libvirtd daemon in /usr/sbin (to override, set $LIBVIRTD_PATH to the name of the libvirtd binary)</code> - you don’t have the libvirtd daemon running or installed, be sure you installed the ‘libvirt-bin’ package and it is running</li> <li><code>Call to virConnectOpen failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': Permission denied</code> - you need to be in the libvirt group to access the socket, do ‘sudo adduser <user> libvirtd’ and then re-login to your session.</li> <li>if each call to vagrant ends up with a segfault, it may be because you still have virtualbox around. if so, remove virtualbox to keep only libvirt + KVM. according to <a href="https://github.com/pradels/vagrant-libvirt/issues/75">https://github.com/pradels/vagrant-libvirt/issues/75</a> having two virtualization engines installed simultaneously can lead to such weird issues.</li> -<li>see the <a href="https://github.com/pradels/vagrant-libvirt/issues">vagrant-libvirt issue list on github</a></li> -<li>be sure to use vagrant-libvirt >= 0.0.11 and sahara >= 0.0.16 (which are the latest stable gems you would get with <code>vagrant plugin install [vagrant-libvirt|sahara]</code>) for proper libvirt support,</li> </ul> diff --git a/docs/en/tutorials/vagrant/index.html b/docs/en/tutorials/vagrant/index.html index 95bd6b71..181a3ccf 100644 --- a/docs/en/tutorials/vagrant/index.html +++ b/docs/en/tutorials/vagrant/index.html @@ -437,12 +437,12 @@ $ leap local save web1 <p>Clone the platform with</p> -<pre><code>git clone --recursive -b develop https://github.com/leapcode/leap_platform.git +<pre><code>git clone https://leap.se/git/leap_platform </code></pre> <p>Start the vagrant box with</p> -<pre><code>cd leap_platform +<pre><code>cd leap_platform/tests/example-provider vagrant up </code></pre> @@ -531,7 +531,7 @@ started by the bitmask client:</p> sudo apt-get install ruby-dev libxslt-dev libxml2-dev libvirt-dev # install the required plugins -vagrant plugin install vagrant-libvirt fog fog-libvirt sahara +vagrant plugin install vagrant-libvirt sahara </code></pre> <p>Log out and then log back in.</p> @@ -585,8 +585,6 @@ virsh pool-autostart vagrant <li><code>Call to virConnectOpen failed: internal error: Unable to locate libvirtd daemon in /usr/sbin (to override, set $LIBVIRTD_PATH to the name of the libvirtd binary)</code> - you don’t have the libvirtd daemon running or installed, be sure you installed the ‘libvirt-bin’ package and it is running</li> <li><code>Call to virConnectOpen failed: Failed to connect socket to '/var/run/libvirt/libvirt-sock': Permission denied</code> - you need to be in the libvirt group to access the socket, do ‘sudo adduser <user> libvirtd’ and then re-login to your session.</li> <li>if each call to vagrant ends up with a segfault, it may be because you still have virtualbox around. if so, remove virtualbox to keep only libvirt + KVM. according to <a href="https://github.com/pradels/vagrant-libvirt/issues/75">https://github.com/pradels/vagrant-libvirt/issues/75</a> having two virtualization engines installed simultaneously can lead to such weird issues.</li> -<li>see the <a href="https://github.com/pradels/vagrant-libvirt/issues">vagrant-libvirt issue list on github</a></li> -<li>be sure to use vagrant-libvirt >= 0.0.11 and sahara >= 0.0.16 (which are the latest stable gems you would get with <code>vagrant plugin install [vagrant-libvirt|sahara]</code>) for proper libvirt support,</li> </ul> diff --git a/docs/index.html b/docs/index.html index 49465a9d..5ef4303c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -152,7 +152,7 @@ Provider Platform - LEAP Platform Documentation <h2><a name="the-leap-command-line-tool"></a>The <code>leap</code> command line tool</h2> -<p>The <code>leap</code> <a href="en/commands.html">command line tool</a> is used by sysadmins to manage everything about a service provider’s infrastructure.</p> +<p>The <code>leap</code> <a href="en/guide/commands.html">command line tool</a> is used by sysadmins to manage everything about a service provider’s infrastructure.</p> <p>Keep these rules in mind:</p> diff --git a/lib/leap_cli/cloud.rb b/lib/leap_cli/cloud.rb index 268cea38..b8e45b3b 100644 --- a/lib/leap_cli/cloud.rb +++ b/lib/leap_cli/cloud.rb @@ -1,4 +1,3 @@ - -require 'fog/aws' +require_relative 'cloud/dependencies.rb' require_relative 'cloud/cloud.rb' require_relative 'cloud/image.rb' diff --git a/lib/leap_cli/cloud/dependencies.rb b/lib/leap_cli/cloud/dependencies.rb index fd690e59..670d6134 100644 --- a/lib/leap_cli/cloud/dependencies.rb +++ b/lib/leap_cli/cloud/dependencies.rb @@ -1,40 +1,37 @@ # -# I am not sure this is a good idea, but it might be. Tricky, so disabled for now +# Ensure that the needed fog gems are installed # - -=begin module LeapCli class Cloud - def self.check_required_gems - begin - require "fog" - rescue LoadError - bail! do - log :error, "The 'vm' command requires the gem 'fog-core'. Please run `gem install fog-core` and try again." - end - end + SUPPORTED = { + 'aws' => {require: 'fog/aws', gem: 'fog-aws'} + }.freeze - fog_gems = @cloud.required_gems - if !options[:mock] && fog_gems.empty? - bail! do - log :warning, "no vm providers are configured in cloud.json." - log "You must have credentials for one of: #{@cloud.possible_apis.join(', ')}." + def self.check_dependencies!(config) + required_gem = map_api_to_gem(config['api']) + if required_gem.nil? + Util.bail! do + Util.log :error, "The API '#{config['api']}' specified in cloud.json is not one that I know how to speak. Try one of #{supported_list}." end end - fog_gems.each do |name, gem_name| - begin - require gem_name.sub('-','/') - rescue LoadError - bail! do - log :error, "The 'vm' command requires the gem '#{gem_name}' (because of what is configured in cloud.json)." - log "Please run `sudo gem install #{gem_name}` and try again." - end + begin + require required_gem[:require] + rescue LoadError + Util.bail! do + Util.log :error, "The 'vm' command requires the gem '#{required_gem[:gem]}'. Please run `gem install #{required_gem[:gem]}` and try again." + Util.log "(make sure you install the gem in the ruby #{RUBY_VERSION} environment)" end end end + def self.supported_list + SUPPORTED.keys.join(', ') + end + + def self.map_api_to_gem(api) + SUPPORTED[api] + end end end -=end
\ No newline at end of file diff --git a/lib/leap_cli/commands/ca.rb b/lib/leap_cli/commands/cert.rb index 3c5fc7d5..68fa9444 100644 --- a/lib/leap_cli/commands/ca.rb +++ b/lib/leap_cli/commands/cert.rb @@ -102,13 +102,13 @@ module LeapCli; module Commands def generate_test_client_cert(prefix=nil) require 'leap_cli/x509' cert = CertificateAuthority::Certificate.new - cert.serial_number.number = cert_serial_number(provider.domain) - cert.subject.common_name = [prefix, random_common_name(provider.domain)].join + cert.serial_number.number = X509.cert_serial_number(provider.domain) + cert.subject.common_name = [prefix, X509.random_common_name(provider.domain)].join cert.not_before = X509.yesterday cert.not_after = X509.yesterday.advance(:years => 1) cert.key_material.generate_key(1024) # just for testing, remember! - cert.parent = client_ca_root - cert.sign! client_test_signing_profile + cert.parent = X509.client_ca_root + cert.sign! X509.client_test_signing_profile yield cert.key_material.private_key.to_pem, cert.to_pem end @@ -281,11 +281,13 @@ module LeapCli; module Commands if status == 'valid' log 'authorized!', color: :green, style: :bold elsif status == 'error' - bail! :error, message + bail! :error, message.inspect elsif status == 'unauthorized' - bail!(:unauthorized, message, color: :yellow, style: :bold) do + bail!(:unauthorized, message.inspect, color: :yellow, style: :bold) do log 'You must first run `leap cert register` to register the account key with letsencrypt.org' end + else + bail!(:error, "unrecognized status: #{status.inspect}, #{message.inspect}") end log :fetching, "new certificate from letsencrypt.org" @@ -335,31 +337,41 @@ module LeapCli; module Commands # This method will bail if any checks fail. # def domain_ready_for_acme!(domain) - begin - uri = URI("https://#{domain}/.well-known/acme-challenge/ok") - options = { - use_ssl: true, - open_timeout: 5, - verify_mode: OpenSSL::SSL::VERIFY_NONE - } - Net::HTTP.start(uri.host, uri.port, options) do |http| - http.request(Net::HTTP::Get.new(uri)) do |response| - if !response.is_a?(Net::HTTPSuccess) - bail!(:error, "Could not GET %s" % uri) do - log "%s %s" % [response.code, response.message] - log "You may need to run `leap deploy`" - end + uri = URI("https://#{domain}/.well-known/acme-challenge/ok") + options = { + use_ssl: true, + open_timeout: 5, + verify_mode: OpenSSL::SSL::VERIFY_NONE + } + http_get(uri, options) + end + + private + + def http_get(uri, options, limit = 10) + raise ArgumentError, "HTTP redirect too deep (#{uri})" if limit == 0 + Net::HTTP.start(uri.host, uri.port, options) do |http| + http.request(Net::HTTP::Get.new(uri)) do |response| + case response + when Net::HTTPSuccess then + return response + when Net::HTTPRedirection then + return http_get(URI(response['location']), options, limit - 1) + else + bail!(:error, "Could not GET %s" % uri) do + log "%s %s" % [response.code, response.message] + log "You may need to run `leap deploy`" end end end - rescue Errno::ETIMEDOUT, Net::OpenTimeout - bail! :error, "Connection attempt timed out: %s" % uri - rescue Interrupt - bail! - rescue StandardError => exc - bail!(:error, "Could not GET %s" % uri) do - log exc.to_s - end + end + rescue Errno::ETIMEDOUT, Net::OpenTimeout + bail! :error, "Connection attempt timed out: %s" % uri + rescue Interrupt + bail! + rescue StandardError => exc + bail!(:error, "Could not GET %s" % uri) do + log exc.to_s end end diff --git a/lib/leap_cli/commands/compile.rb b/lib/leap_cli/commands/compile.rb index 92c879d7..16dff3df 100644 --- a/lib/leap_cli/commands/compile.rb +++ b/lib/leap_cli/commands/compile.rb @@ -155,7 +155,7 @@ module LeapCli buffer = StringIO.new keys = Dir.glob(path([:user_ssh, '*'])) if keys.empty? - bail! "You must have at least one public SSH user key configured in order to proceed. See `leap help add-user`." + bail! "You must have at least one public SSH user key configured in order to proceed. See `leap help user add`." end if file_exists?(path(:monitor_pub_key)) keys << path(:monitor_pub_key) diff --git a/lib/leap_cli/commands/node.rb b/lib/leap_cli/commands/node.rb index 60540de9..9cde15bc 100644 --- a/lib/leap_cli/commands/node.rb +++ b/lib/leap_cli/commands/node.rb @@ -45,6 +45,23 @@ module LeapCli; module Commands do_node_rm(global_options, options, args) end end + + node.desc 'Mark a node as disabled.' + node.arg_name 'NAME' + node.command :disable do |cmd| + cmd.action do |global_options,options,args| + do_node_disable(global_options, options, args) + end + end + + node.desc 'Mark a node as enabled.' + node.arg_name 'NAME' + node.command :enable do |cmd| + cmd.action do |global_options,options,args| + do_node_enable(global_options, options, args) + end + end + end ## @@ -126,4 +143,14 @@ module LeapCli; module Commands remove_node_facts(node.name) end + def do_node_enable(global, options, args) + node = get_node_from_args(args, include_disabled: true) + node.update_json({}, remove: ["enabled"]) + end + + def do_node_disable(global, options, args) + node = get_node_from_args(args, include_disabled: true) + node.update_json("enabled" => false) + end + end; end diff --git a/lib/leap_cli/commands/ping.rb b/lib/leap_cli/commands/ping.rb new file mode 100644 index 00000000..4283d9b3 --- /dev/null +++ b/lib/leap_cli/commands/ping.rb @@ -0,0 +1,58 @@ +module LeapCli; module Commands + + desc "Ping nodes to see if they are alive." + long_desc "Attempts to ping each node in the FILTER set." + arg_name "FILTER" + command :ping do |c| + c.flag 'timeout', :arg_name => "TIMEOUT", + :default_value => 2, :desc => 'Wait at most TIMEOUT seconds.' + c.flag 'count', :arg_name => "COUNT", + :default_value => 2, :desc => 'Ping COUNT times.' + c.action do |global, options, args| + do_ping(global, options, args) + end + end + + private + + def do_ping(global, options, args) + assert_bin!('ping') + + timeout = [options[:timeout].to_i, 1].max + count = [options[:count].to_i, 1].max + nodes = nil + + if args && args.any? + node = manager.disabled_node(args.first) + if node + nodes = Config::ObjectList.new + nodes.add(node.name, node) + end + end + + nodes ||= manager.filter! args + + threads = [] + nodes.each_node do |node| + threads << Thread.new do + cmd = "ping -i 0.2 -n -q -W #{timeout} -c #{count} #{node.ip_address} 2>&1" + log(2, cmd) + output = `#{cmd}` + if $?.success? + last = output.split("\n").last + times = last.split('=').last.strip + min, avg, max, mdev = times.split('/') + log("ping #{min} ms", host: node.name, color: :green) + else + log(:failed, "to ping #{node.ip_address}", host: node.name) + end + end + end + threads.map(&:join) + + log("done") + end + +end; end + + diff --git a/lib/leap_cli/commands/run.rb b/lib/leap_cli/commands/run.rb index cad9b7a0..9149d594 100644 --- a/lib/leap_cli/commands/run.rb +++ b/lib/leap_cli/commands/run.rb @@ -3,13 +3,27 @@ module LeapCli; module Commands desc 'Run a shell command remotely' long_desc "Runs the specified command COMMAND on each node in the FILTER set. " + "For example, `leap run 'uname -a' webapp`" - arg_name 'COMMAND FILTER' command :run do |c| c.switch 'stream', :default => false, :desc => 'If set, stream the output as it arrives. (default: --stream for a single node, --no-stream for multiple nodes)' c.flag 'port', :arg_name => 'SSH_PORT', :desc => 'Override default SSH port used when trying to connect to the server.' - c.action do |global, options, args| - run_shell_command(global, options, args) + + c.desc 'Run an arbitrary shell command.' + c.arg_name 'FILTER', optional: true + c.command :command do |command| + command.action do |global, options, args| + run_shell_command(global, options, args) + end + end + + c.desc 'Generate one or more new invite codes.' + c.arg_name '[COUNT] [ENVIRONMENT]' + c.command :invite do |invite| + invite.action do |global_options,options,args| + run_new_invites(global_options, options, args) + end end + + c.default_command :command end private @@ -27,6 +41,39 @@ module LeapCli; module Commands end end + CMD_NEW_INVITES="cd /srv/leap/webapp; RAILS_ENV=production bundle exec rake \"generate_invites[NUM,USES]\"" + + def run_new_invites(global, options, args) + require 'leap_cli/ssh' + count = 1 + uses = 1 + env = nil + arg1 = args.shift + arg2 = args.shift + if arg1 && arg2 + env = manager.env(arg2) + count = arg1 + elsif arg1 + env = manager.env(arg1) + else + env = manager.env(nil) + end + unless env + bail! "Environment name you specified does not match one that is available. See `leap env ls` for the available names" + end + + env_name = env.name == 'default' ? nil : env.name + webapp_nodes = env.nodes[:environment => env_name][:services => 'webapp'].first + if webapp_nodes.empty? + bail! "Could not find a webapp node for the specified environment" + end + stream_command( + webapp_nodes, + CMD_NEW_INVITES.sub('NUM', count.to_s).sub('USES', uses.to_s), + options + ) + end + def capture_command(nodes, cmd, options) SSH.remote_command(nodes, options) do |ssh, host| output = ssh.capture(cmd, :log_output => false) diff --git a/lib/leap_cli/commands/test.rb b/lib/leap_cli/commands/test.rb index 70eb00fd..e2815aae 100644 --- a/lib/leap_cli/commands/test.rb +++ b/lib/leap_cli/commands/test.rb @@ -35,7 +35,7 @@ module LeapCli; module Commands SSH::remote_command(node, options) do |ssh, host| ssh.stream(test_cmd(options), :raise_error => true, :log_wrap => true) end - rescue LeapCli::SSH::ExecuteError + rescue LeapCli::SSH::TimeoutError, SSHKit::Runner::ExecuteError, SSHKit::Command::Failed if options[:continue] exit_status(1) else diff --git a/lib/leap_cli/commands/user.rb b/lib/leap_cli/commands/user.rb index 1ca92719..7fd5f52d 100644 --- a/lib/leap_cli/commands/user.rb +++ b/lib/leap_cli/commands/user.rb @@ -113,14 +113,42 @@ module LeapCli def do_list_users(global, options, args) require 'leap_cli/ssh' + ssh_keys = {} + Dir.glob("#{ENV['HOME']}/.ssh/*.pub").each do |keyfile| + key = SSH::Key.load(keyfile) + ssh_keys[key.fingerprint] = key if key + end + + ssh_agent_keys = {} + if !`which ssh-add`.empty? + `ssh-add -L`.split("\n").each do |keystring| + key = SSH::Key.load(keystring) + ssh_agent_keys[key.fingerprint] = key if key + end + end + Dir.glob(path([:user_ssh, '*'])).each do |keyfile| username = File.basename(File.dirname(keyfile)) log username, :color => :cyan do log Path.relative_path(keyfile) key = SSH::Key.load(keyfile) - log 'SSH MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) - log 'SSH SHA256 fingerprint: ' + key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) - log 'DER MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + if key.nil? + log :warning, "could not read ssh key #{keyfile}" do + log "currently, only these ssh key types are supported: " + SSH::Key::SUPPORTED_TYPES.join(", ") + end + else + log 'SSH MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :ssh, :encoding => :hex) + log 'SSH SHA256 fingerprint: ' + key.fingerprint(:digest => :sha256, :type => :ssh, :encoding => :base64) + log 'DER MD5 fingerprint: ' + key.fingerprint(:digest => :md5, :type => :der, :encoding => :hex) + if ssh_keys[key.fingerprint] + log 'Matches local key: ' + ssh_keys[key.fingerprint].filename, color: :green + if ssh_agent_keys[key.fingerprint] + log 'Matches ssh-agent key: ' + ssh_agent_keys[key.fingerprint].summary(encoding: :base64), color: :green + else + log :error, 'No matching key in the ssh-agent' + end + end + end end end end @@ -154,6 +182,9 @@ module LeapCli end else key_index = 0 + log "Picking the only compatible ssh key: "+ ssh_keys[key_index].filename do + log ssh_keys[key_index].summary + end end return ssh_keys[key_index] diff --git a/lib/leap_cli/commands/vagrant.rb b/lib/leap_cli/commands/vagrant.rb index f8a75b61..78b2fede 100644 --- a/lib/leap_cli/commands/vagrant.rb +++ b/lib/leap_cli/commands/vagrant.rb @@ -132,10 +132,10 @@ module LeapCli; module Commands lines << %[ config.vm.provider "virtualbox" do |v|] lines << %[ v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]] lines << %[ v.name = "#{node.name}"] - lines << %[ v.memory = 1536] + lines << %[ v.memory = 2048] lines << %[ end] lines << %[ config.vm.provider "libvirt" do |v|] - lines << %[ v.memory = 1536] + lines << %[ v.memory = 2048] lines << %[ end] lines << %[ #{leapfile.custom_vagrant_vm_line}] if leapfile.custom_vagrant_vm_line lines << %[ end] diff --git a/lib/leap_cli/commands/vm.rb b/lib/leap_cli/commands/vm.rb index 790774f1..6f97dbce 100644 --- a/lib/leap_cli/commands/vm.rb +++ b/lib/leap_cli/commands/vm.rb @@ -415,7 +415,6 @@ module LeapCli; module Commands config = manager.env.cloud name = nil if options[:mock] - Fog.mock! name = 'mock_aws' config['mock_aws'] = { "api" => "aws", @@ -451,6 +450,10 @@ module LeapCli; module Commands assert! entry['api'] == 'aws', "cloud.json: currently, only 'aws' is supported for `api`." assert! entry['vendor'] == 'aws', "cloud.json: currently, only 'aws' is supported for `vendor`." + LeapCli::Cloud::check_dependencies!(entry) + if options[:mock] + Fog.mock! + end return LeapCli::Cloud.new(name, entry, node) end diff --git a/lib/leap_cli/config/environment.rb b/lib/leap_cli/config/environment.rb index ce570839..0410ef5b 100644 --- a/lib/leap_cli/config/environment.rb +++ b/lib/leap_cli/config/environment.rb @@ -122,14 +122,25 @@ module LeapCli; module Config end # - # Alters the node's json config file. Unfortunately, doing this will - # strip out all the comments. + # Alters the node's json config file. As a side effect, all comments get + # moved to the top of the file. # - def update_node_json(node, new_values) + # NOTE: This does a shallow merge! In other words, a call like this... + # + # update_node_json(node, {"webapp" => {"domain" => "example.org"}) + # + # ...is probably not what you want, because it will entirely remove all + # existing entries under "webapp". + # + def update_node_json(node, new_values, options=nil) node_json_path = Path.named_path([:node_config, node.name]) + comments = read_comments(node_json_path) old_data = load_json(node_json_path, Config::Node) + options && options[:remove] && options[:remove].each do |key| + old_data.delete(key) + end new_data = old_data.merge(new_values) - new_contents = JSON.sorted_generate(new_data) + "\n" + new_contents = [comments, JSON.sorted_generate(new_data), "\n"].join Util::write_file! node_json_path, new_contents end @@ -152,6 +163,17 @@ module LeapCli; module Config results end + def read_comments(filename) + buffer = StringIO.new + File.open(filename, "rb", :encoding => 'UTF-8') do |f| + while (line = f.gets) + next unless line =~ /^\s*\/\// + buffer << line + end + end + return buffer.string.force_encoding('utf-8') + end + def load_json(filename, object_class, options={}) if !File.exist?(filename) return object_class.new(self) diff --git a/lib/leap_cli/config/manager.rb b/lib/leap_cli/config/manager.rb index d69a5808..a9f1a85f 100644 --- a/lib/leap_cli/config/manager.rb +++ b/lib/leap_cli/config/manager.rb @@ -342,14 +342,25 @@ module LeapCli if node.vagrant? return self.env("local") else - environment = self.env(default_environment) + environment = nil if node['tags'] node['tags'].to_a.each do |tag| if self.environment_names.include?(tag) - environment = self.env(tag) + if environment.nil? + environment = self.env(tag) + else + LeapCli::Util.bail! do + LeapCli.log( + :error, + "The node '%s' is invalid, because it cannot have two environments ('%s' and '%s')." % + [node.name, environment.name, tag] + ) + end + end end end end + environment ||= self.env(default_environment) return environment end end diff --git a/lib/leap_cli/config/node.rb b/lib/leap_cli/config/node.rb index 23abdee3..a7c5c1e4 100644 --- a/lib/leap_cli/config/node.rb +++ b/lib/leap_cli/config/node.rb @@ -169,8 +169,8 @@ module LeapCli; module Config # # modifies the config file nodes/NAME.json for this node. # - def update_json(new_values) - self.env.update_node_json(node, new_values) + def update_json(new_values, options=nil) + self.env.update_node_json(node, new_values, options) end # diff --git a/lib/leap_cli/config/object_list.rb b/lib/leap_cli/config/object_list.rb index 80f89d92..815864e4 100644 --- a/lib/leap_cli/config/object_list.rb +++ b/lib/leap_cli/config/object_list.rb @@ -49,6 +49,10 @@ module LeapCli end end + def first + ObjectList.new(self.values.first) + end + def exclude(node) list = self.dup list.delete(node.name) diff --git a/lib/leap_cli/macros/haproxy.rb b/lib/leap_cli/macros/haproxy.rb deleted file mode 100644 index 3fef24c4..00000000 --- a/lib/leap_cli/macros/haproxy.rb +++ /dev/null @@ -1,73 +0,0 @@ -# encoding: utf-8 - -## -## HAPROXY -## - -module LeapCli - module Macro - - # - # creates a hash suitable for configuring haproxy. the key is the node name of the server we are proxying to. - # - # * node_list - a hash of nodes for the haproxy servers - # * stunnel_client - contains the mappings to local ports for each server node. - # * non_stunnel_port - in case self is included in node_list, the port to connect to. - # - # 1000 weight is used for nodes in the same location. - # 100 otherwise. - # - def haproxy_servers(node_list, stunnel_clients, non_stunnel_port=nil) - default_weight = 10 - local_weight = 100 - - # record the hosts_file - hostnames(node_list) - - # create a simple map for node name -> local stunnel accept port - accept_ports = stunnel_clients.inject({}) do |hsh, stunnel_entry| - name = stunnel_entry.first.sub(/_[0-9]+$/, '') - hsh[name] = stunnel_entry.last['accept_port'] - hsh - end - - # if one the nodes in the node list is ourself, then there will not be a stunnel to it, - # but we need to include it anyway in the haproxy config. - if node_list[self.name] && non_stunnel_port - accept_ports[self.name] = non_stunnel_port - end - - # create the first pass of the servers hash - servers = node_list.values.inject(Config::ObjectList.new) do |hsh, node| - # make sure we have a port to talk to - unless accept_ports[node.name] - error "haproxy needs a local port to talk to when connecting to #{node.name}" - end - weight = default_weight - try { - weight = local_weight if self.location.name == node.location.name - } - hsh[node.name] = Config::Object[ - 'backup', false, - 'host', 'localhost', - 'port', accept_ports[node.name], - 'weight', weight - ] - if node.services.include?('couchdb') - hsh[node.name]['writable'] = node.couch.mode != 'mirror' - end - hsh - end - - # if there are some local servers, make the others backup - if servers.detect{|k,v| v.weight == local_weight} - servers.each do |k,server| - server['backup'] = server['weight'] == default_weight - end - end - - return servers - end - - end -end diff --git a/lib/leap_cli/macros/stunnel.rb b/lib/leap_cli/macros/stunnel.rb index 821bda38..59a38fad 100644 --- a/lib/leap_cli/macros/stunnel.rb +++ b/lib/leap_cli/macros/stunnel.rb @@ -87,6 +87,18 @@ module LeapCli } end + # + # what it the port of the couchdb we should connect to. + # host will always be localhost. + # + def couchdb_port + if services.include?('couchdb') + couch.port + else + stunnel.clients.couch_client.values.first.accept_port + end + end + private # @@ -103,4 +115,4 @@ module LeapCli end end -end
\ No newline at end of file +end diff --git a/lib/leap_cli/ssh/backend.rb b/lib/leap_cli/ssh/backend.rb index 3894d815..599fc9a0 100644 --- a/lib/leap_cli/ssh/backend.rb +++ b/lib/leap_cli/ssh/backend.rb @@ -178,7 +178,7 @@ module LeapCli rescue StandardError => exc if exc.is_a?(SSHKit::Command::Failed) || exc.is_a?(SSHKit::Runner::ExecuteError) if @options[:raise_error] - raise LeapCli::SSH::ExecuteError, exc.to_s + raise exc elsif @options[:fail_msg] @logger.log(@options[:fail_msg], host: @host.hostname, :color => :red) else diff --git a/lib/leap_cli/ssh/key.rb b/lib/leap_cli/ssh/key.rb index 76223b7e..108b6137 100644 --- a/lib/leap_cli/ssh/key.rb +++ b/lib/leap_cli/ssh/key.rb @@ -254,9 +254,9 @@ module LeapCli end if digest == "MD5" && encoding == :hex - return fp.scan(/../).join(':') + return fp.strip.scan(/../).join(':') else - return fp + return fp.strip end end @@ -267,11 +267,12 @@ module LeapCli Net::SSH::Buffer.from(:key, @key).to_s.split("\001\000").last.size * 8 end - def summary + def summary(type: :ssh, digest: :sha256, encoding: :hex) + fp = digest.to_s.upcase + ":" + self.fingerprint(type: type, digest: digest, encoding: encoding) if self.filename - "%s %s %s (%s)" % [self.type, self.bits, self.fingerprint, File.basename(self.filename)] + "%s %s %s (%s)" % [self.type, self.bits, fp, File.basename(self.filename)] else - "%s %s %s" % [self.type, self.bits, self.fingerprint] + "%s %s %s" % [self.type, self.bits, fp] end end diff --git a/platform.rb b/platform.rb index 2ff0a27f..935fa385 100644 --- a/platform.rb +++ b/platform.rb @@ -4,7 +4,7 @@ # Leap::Platform.define do - self.version = "0.9" + self.version = "0.10" self.compatible_cli = "1.9".."1.99" # diff --git a/provider_base/common.json b/provider_base/common.json index 893d5daf..1052b8e1 100644 --- a/provider_base/common.json +++ b/provider_base/common.json @@ -12,7 +12,8 @@ "name": "= node.name + '.' + (dns.public ? domain.full_suffix : domain.internal_suffix)" }, "dns": { - "public": "= service_type != 'internal_service'" + "public": "= service_type != 'internal_service'", + "aliases": [] }, "ssh": { "authorized_keys": "= authorized_keys", @@ -71,11 +72,12 @@ "nickserver": { "type": "git", "source": "https://leap.se/git/nickserver", - "revision": "origin/version/0.9" + "revision": "tags/0.10.0" }, "platform": { "apt": { - "basic": "= 'http://deb.leap.se/' + Leap::Platform.major_version" + "basic": "http://deb.leap.se/platform", + "component": "0.10" } }, "soledad": { @@ -86,7 +88,7 @@ "webapp": { "type": "git", "source": "https://leap.se/git/leap_web", - "revision": "origin/version/0.8" + "revision": "tags/0.9.2" } } } diff --git a/provider_base/provider.json b/provider_base/provider.json index 81b2ea98..a9980bf8 100644 --- a/provider_base/provider.json +++ b/provider_base/provider.json @@ -58,7 +58,7 @@ } }, "client_version": { - "min": "0.7", + "min": "0.9.2", "max": null } } diff --git a/provider_base/services/_tor_common.json b/provider_base/services/_tor_common.json new file mode 100644 index 00000000..461232dc --- /dev/null +++ b/provider_base/services/_tor_common.json @@ -0,0 +1,8 @@ +{ + "tor": { + "type": "disabled", + "contacts": "= [provider.contacts['tor'] || provider.contacts.default].flatten", + "nickname": "= (self.name + secret(:tor_family)).sub('_','')[0..18]", + "family": "= nodes[:services => 'tor'][:environment => '!local'].field('tor.nickname').join(',')" + } +} diff --git a/provider_base/services/mx.json b/provider_base/services/mx.json index c7e99d85..480d7c6e 100644 --- a/provider_base/services/mx.json +++ b/provider_base/services/mx.json @@ -19,17 +19,12 @@ "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" } }, - "haproxy": { - "couch": { - "listen_port": 4096, - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" - } - }, "couchdb_leap_mx_user": { "username": "= global.services[:couchdb].couch.users[:leap_mx].username", "password": "= secret :couch_leap_mx_password", "salt": "= hex_secret :couch_leap_mx_password_salt, 128" }, + "couchdb_port": "= couchdb_port", "mynetworks": "= host_ips(nodes)", "rbls": ["zen.spamhaus.org"], "clamav": { @@ -37,7 +32,6 @@ }, "x509": { "use": true, - "use_commercial": true, "ca_cert": "= file :ca_cert, :missing => 'provider CA. Run `leap cert ca`'", "client_ca_cert": "= file :client_ca_cert, :missing => 'Certificate Authority. Run `leap cert ca`'", "client_ca_key": "= file :client_ca_key, :missing => 'Certificate Authority. Run `leap cert ca`'" diff --git a/provider_base/services/static.rb b/provider_base/services/static.rb new file mode 100644 index 00000000..4c7d2e59 --- /dev/null +++ b/provider_base/services/static.rb @@ -0,0 +1,4 @@ +if self['static'] && self['static']['domains'] + self['dns']['aliases'] += self['static']['domains'].keys + self['dns']['aliases'].uniq! +end
\ No newline at end of file diff --git a/provider_base/services/tor.json b/provider_base/services/tor.json index e80310fe..a0d44fef 100644 --- a/provider_base/services/tor.json +++ b/provider_base/services/tor.json @@ -9,7 +9,8 @@ "key_type": "RSA", "public_key": "= tor_public_key_path(:node_tor_pub_key, tor.hidden_service.key_type) if tor.hidden_service.active", "private_key": "= tor_private_key_path(:node_tor_priv_key, tor.hidden_service.key_type) if tor.hidden_service.active", - "address": "=> tor.hidden_service.active && onion_address(:node_tor_pub_key)" + "address": "=> tor.hidden_service.active && onion_address(:node_tor_pub_key)", + "single_hop": false } } } diff --git a/provider_base/services/tor_exit.json b/provider_base/services/tor_exit.json new file mode 100644 index 00000000..dab3b76f --- /dev/null +++ b/provider_base/services/tor_exit.json @@ -0,0 +1,5 @@ +{ + "tor": { + "bandwidth_rate": 6550 + } +} diff --git a/provider_base/services/tor_exit.rb b/provider_base/services/tor_exit.rb new file mode 100644 index 00000000..bd801a3d --- /dev/null +++ b/provider_base/services/tor_exit.rb @@ -0,0 +1,6 @@ +if self.services.include?("tor_hidden_service") || self.services.include?("tor_relay") + LeapCli.log :error, "service `tor_exit` is not compatible with tor_relay or tor_hidden_service (node #{self.name})." + exit(1) +end +apply_partial("_tor_common") +self.tor['type'] = "exit" diff --git a/provider_base/services/tor_hidden_service.json b/provider_base/services/tor_hidden_service.json new file mode 100644 index 00000000..d7f3ec27 --- /dev/null +++ b/provider_base/services/tor_hidden_service.json @@ -0,0 +1,12 @@ +{ + "tor": { + "hidden_service": { + "key_type": "RSA", + "public_key": "= tor_public_key_path(:node_tor_pub_key, tor.hidden_service.key_type)", + "private_key": "= tor_private_key_path(:node_tor_priv_key, tor.hidden_service.key_type)", + "address": "=> onion_address(:node_tor_pub_key)", + "single_hop": false, + "v3": false + } + } +} diff --git a/provider_base/services/tor_hidden_service.rb b/provider_base/services/tor_hidden_service.rb new file mode 100644 index 00000000..8b8eb24d --- /dev/null +++ b/provider_base/services/tor_hidden_service.rb @@ -0,0 +1,4 @@ +if self.services.include?("tor_exit") || self.services.include?("tor_relay") + LeapCli.log :error, "service `tor_hidden_service` is not compatible with tor_exit or tor_relay (node #{self.name})." +end +self.tor['type'] = "hidden_service" diff --git a/provider_base/services/tor_relay.json b/provider_base/services/tor_relay.json new file mode 100644 index 00000000..dab3b76f --- /dev/null +++ b/provider_base/services/tor_relay.json @@ -0,0 +1,5 @@ +{ + "tor": { + "bandwidth_rate": 6550 + } +} diff --git a/provider_base/services/tor_relay.rb b/provider_base/services/tor_relay.rb new file mode 100644 index 00000000..7fce6ae4 --- /dev/null +++ b/provider_base/services/tor_relay.rb @@ -0,0 +1,6 @@ + +if self.services.include?("tor_exit") || self.services.include?("tor_hidden_service") + LeapCli.log :error, "service `tor_relay` is not compatible with tor_exit or tor_hidden_service (node #{self.name})." +end +apply_partial("_tor_common") +self.tor['type'] = "relay" diff --git a/provider_base/services/webapp.json b/provider_base/services/webapp.json index feca9524..ac58ac12 100644 --- a/provider_base/services/webapp.json +++ b/provider_base/services/webapp.json @@ -11,6 +11,7 @@ "tickets", "vmail", "www-data"], "domain": "= provider.domain", "modules": ["user", "billing", "help"], + "couchdb_port": "= couchdb_port", "couchdb_webapp_user": "= global.services[:couchdb].couch.users[:webapp]", "couchdb_admin_user": "= global.services[:couchdb].couch.users[:admin]", "customization_dir": "= file_path 'webapp'", @@ -22,6 +23,7 @@ "invite_required": "= provider.enrollment_policy == 'invite'", "default_service_level": "= provider.service.default_service_level", "service_levels": "= service_levels()", + "secret_key_base": "= secret :webapp_secret_key_base", "secret_token": "= secret :webapp_secret_token", "api_version": 1, "secure": false, @@ -45,12 +47,6 @@ "couch_client": "= stunnel_client(nodes_like_me[:services => :couchdb], global.services[:couchdb].couch.port)" } }, - "haproxy": { - "couch": { - "listen_port": 4096, - "servers": "= haproxy_servers(nodes_like_me[:services => :couchdb], stunnel.clients.couch_client, global.services[:couchdb].couch.port)" - } - }, "definition_files": { "provider": "= file :provider_json_template", "eip_service": "= file [:eip_service_json_template, 'v'+webapp.api_version.to_s]", @@ -67,6 +63,7 @@ }, "nickserver": { "domain": "= 'nicknym.' + domain.full_suffix", + "couchdb_port": "= couchdb_port", "couchdb_nickserver_user": { "username": "= global.services[:couchdb].couch.users[:nickserver].username", "password": "= secret :couch_nickserver_password", @@ -74,9 +71,6 @@ }, "port": 6425 }, - "dns": { - "aliases": "= [domain.full, webapp.domain, api.domain, nickserver.domain]" - }, "x509": { "use": true, "use_commercial": true, diff --git a/provider_base/services/webapp.rb b/provider_base/services/webapp.rb new file mode 100644 index 00000000..a5f10a2d --- /dev/null +++ b/provider_base/services/webapp.rb @@ -0,0 +1,2 @@ +self['dns']['aliases'] += [domain.full, webapp.domain, api.domain, nickserver.domain] +self['dns']['aliases'].uniq! diff --git a/provider_base/templates/provider.json b/provider_base/templates/provider.json new file mode 100644 index 00000000..297327d1 --- /dev/null +++ b/provider_base/templates/provider.json @@ -0,0 +1,19 @@ +// +// This file defines global aspects of your service provider +// See https://leap.se/provider-configuration +// +{ + "domain": "<%= domain %>", + "name": { + "en": "<%= name %>" + }, + "description": { + "en": "You really should change this text" + }, + "contacts": { + "default": "<%= contacts %>" + }, + "languages": ["en"], + "default_language": "en", + "enrollment_policy": "invite" +} diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index 3bf6a5c1..1f80c47c 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -44,8 +44,16 @@ node default { include site_nagios } - if member($services, 'tor') { - include site_tor + if member($services, 'tor_relay') { + include site_tor::relay + } + + if member($services, 'tor_exit') { + include site_tor::relay + } + + if member($services, 'tor_hidden_service') { + include site_tor::hidden_service } if member($services, 'mx') { diff --git a/puppet/modules/haproxy/.fixtures.yml b/puppet/modules/haproxy/.fixtures.yml deleted file mode 100644 index 8d6f22d6..00000000 --- a/puppet/modules/haproxy/.fixtures.yml +++ /dev/null @@ -1,5 +0,0 @@ -fixtures: - repositories: - concat: "git://github.com/ripienaar/puppet-concat.git" - symlinks: - haproxy: "#{source_dir}" diff --git a/puppet/modules/haproxy/.gemfile b/puppet/modules/haproxy/.gemfile deleted file mode 100644 index 9aad840c..00000000 --- a/puppet/modules/haproxy/.gemfile +++ /dev/null @@ -1,5 +0,0 @@ -source :rubygems - -puppetversion = ENV.key?('PUPPET_VERSION') ? "= #{ENV['PUPPET_VERSION']}" : ['>= 2.7'] -gem 'puppet', puppetversion -gem 'puppetlabs_spec_helper', '>= 0.1.0' diff --git a/puppet/modules/haproxy/.gitrepo b/puppet/modules/haproxy/.gitrepo deleted file mode 100644 index ed92831a..00000000 --- a/puppet/modules/haproxy/.gitrepo +++ /dev/null @@ -1,11 +0,0 @@ -; DO NOT EDIT (unless you know what you are doing) -; -; This subdirectory is a git "subrepo", and this file is maintained by the -; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme -; -[subrepo] - remote = https://leap.se/git/puppet_haproxy - branch = master - commit = af322a73c013f80a958ab7d5d31d0c75cf6d0523 - parent = 04279dd8d1390d61d696d2c14817199304ccd4d8 - cmdver = 0.3.0 diff --git a/puppet/modules/haproxy/.travis.yml b/puppet/modules/haproxy/.travis.yml deleted file mode 100644 index fdbc95dc..00000000 --- a/puppet/modules/haproxy/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: ruby -rvm: - - 1.8.7 - - 1.9.3 -script: "rake spec" -branches: - only: - - master -env: - - PUPPET_VERSION=2.6.17 - - PUPPET_VERSION=2.7.19 - #- PUPPET_VERSION=3.0.1 # Breaks due to rodjek/rspec-puppet#58 -notifications: - email: false -gemfile: .gemfile -matrix: - exclude: - - rvm: 1.9.3 - gemfile: .gemfile - env: PUPPET_VERSION=2.6.17 - - rvm: 1.8.7 - gemfile: .gemfile - env: PUPPET_VERSION=3.0.1 diff --git a/puppet/modules/haproxy/CHANGELOG b/puppet/modules/haproxy/CHANGELOG deleted file mode 100644 index 0b6d670f..00000000 --- a/puppet/modules/haproxy/CHANGELOG +++ /dev/null @@ -1,5 +0,0 @@ -2012-10-12 - Version 0.2.0 -- Initial public release -- Backwards incompatible changes all around -- No longer needs ordering passed for more than one listener -- Accepts multiple listen ips/ports/server_names diff --git a/puppet/modules/haproxy/Modulefile b/puppet/modules/haproxy/Modulefile deleted file mode 100644 index e729739b..00000000 --- a/puppet/modules/haproxy/Modulefile +++ /dev/null @@ -1,12 +0,0 @@ -name 'puppetlabs-haproxy' -version '0.2.0' -source 'git://github.com/puppetlabs/puppetlabs-haproxy' -author 'Puppet Labs' -license 'Apache License, Version 2.0' -summary 'Haproxy Module' -description 'An Haproxy module for Redhat family OSes using Storeconfigs' -project_page 'http://github.com/puppetlabs/puppetlabs-haproxy' - -## Add dependencies, if any: -# dependency 'username/name', '>= 1.2.0' -dependency 'ripienaar/concat', '>= 0.1.0' diff --git a/puppet/modules/haproxy/README.md b/puppet/modules/haproxy/README.md deleted file mode 100644 index d209e9ab..00000000 --- a/puppet/modules/haproxy/README.md +++ /dev/null @@ -1,87 +0,0 @@ -PuppetLabs Module for haproxy -============================= - -HAProxy is an HA proxying daemon for load-balancing to clustered services. It -can proxy TCP directly, or other kinds of traffic such as HTTP. - -Dependencies ------------- - -Tested and built on Debian, Ubuntu and CentOS - -Currently requires the ripienaar/concat module on the Puppet Forge and uses storeconfigs on the Puppet Master to export/collect resources -from all balancer members. - -Basic Usage ------------ - -This haproxy uses storeconfigs to collect and realize balancer member servers -on a load balancer server. - -*To install and configure HAProxy server listening on port 8140* - -```puppet -node 'haproxy-server' { - class { 'haproxy': } - haproxy::listen { 'puppet00': - ipaddress => $::ipaddress, - ports => '8140', - } -} -``` - -*To add backend loadbalance members* - -```puppet -node 'webserver01' { - @@haproxy::balancermember { $fqdn: - listening_service => 'puppet00', - server_names => $::hostname, - ipaddresses => $::ipaddress, - ports => '8140', - options => 'check' - } -} -``` - -Configuring haproxy options ---------------------------- - -The base `haproxy` class can accept two parameters which will configure basic -behaviour of the haproxy server daemon: - -- `global_options` to configure the `global` section in `haproxy.cfg` -- `defaults_options` to configure the `defaults` section in `haproxy.cfg` - -Configuring haproxy daemon listener ------------------------------------ - -One `haproxy::listen` defined resource should be defined for each HAProxy loadbalanced set of backend servers. The title of the `haproxy::listen` resource is the key to which balancer members will be proxied to. The `ipaddress` field should be the public ip address which the loadbalancer will be contacted on. The `ports` attribute can accept an array or comma-separated list of ports which should be proxied to the `haproxy::balancermemeber` nodes. - -Configuring haproxy loadbalanced member nodes ---------------------------------------------- - -The `haproxy::balacemember` defined resource should be exported from each node -which is serving loadbalanced traffic. the `listening_service` attribute will -associate it with `haproxy::listen` directives on the haproxy node. -`ipaddresses` and `ports` will be assigned to the member to be contacted on. If an array of `ipaddresses` and `server_names` are provided then they will be added to the config in lock-step. - - -Copyright and License ---------------------- - -Copyright (C) 2012 [Puppet Labs](https://www.puppetlabs.com/) Inc - -Puppet Labs can be contacted at: info@puppetlabs.com - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/puppet/modules/haproxy/Rakefile b/puppet/modules/haproxy/Rakefile deleted file mode 100644 index cd3d3799..00000000 --- a/puppet/modules/haproxy/Rakefile +++ /dev/null @@ -1 +0,0 @@ -require 'puppetlabs_spec_helper/rake_tasks' diff --git a/puppet/modules/haproxy/manifests/balancermember.pp b/puppet/modules/haproxy/manifests/balancermember.pp deleted file mode 100644 index a0e27539..00000000 --- a/puppet/modules/haproxy/manifests/balancermember.pp +++ /dev/null @@ -1,95 +0,0 @@ -# == Define Resource Type: haproxy::balancermember -# -# This type will setup a balancer member inside a listening service -# configuration block in /etc/haproxy/haproxy.cfg on the load balancer. -# currently it only has the ability to specify the instance name, -# ip address, port, and whether or not it is a backup. More features -# can be added as needed. The best way to implement this is to export -# this resource for all haproxy balancer member servers, and then collect -# them on the main haproxy load balancer. -# -# === Requirement/Dependencies: -# -# Currently requires the ripienaar/concat module on the Puppet Forge and -# uses storeconfigs on the Puppet Master to export/collect resources -# from all balancer members. -# -# === Parameters -# -# [*name*] -# The title of the resource is arbitrary and only utilized in the concat -# fragment name. -# -# [*listening_service*] -# The haproxy service's instance name (or, the title of the -# haproxy::listen resource). This must match up with a declared -# haproxy::listen resource. -# -# [*ports*] -# An array or commas-separated list of ports for which the balancer member -# will accept connections from the load balancer. Note that cookie values -# aren't yet supported, but shouldn't be difficult to add to the -# configuration. If you use an array in server_names and ipaddresses, the -# same port is used for all balancermembers. -# -# [*server_names*] -# The name of the balancer member server as known to haproxy in the -# listening service's configuration block. This defaults to the -# hostname. Can be an array of the same length as ipaddresses, -# in which case a balancermember is created for each pair of -# server_names and ipaddresses (in lockstep). -# -# [*ipaddresses*] -# The ip address used to contact the balancer member server. -# Can be an array, see documentation to server_names. -# -# [*options*] -# An array of options to be specified after the server declaration -# in the listening service's configuration block. -# -# -# === Examples -# -# Exporting the resource for a balancer member: -# -# @@haproxy::balancermember { 'haproxy': -# listening_service => 'puppet00', -# ports => '8140', -# server_names => $::hostname, -# ipaddresses => $::ipaddress, -# options => 'check', -# } -# -# -# Collecting the resource on a load balancer -# -# Haproxy::Balancermember <<| listening_service == 'puppet00' |>> -# -# Creating the resource for multiple balancer members at once -# (for single-pass installation of haproxy without requiring a first -# pass to export the resources if you know the members in advance): -# -# haproxy::balancermember { 'haproxy': -# listening_service => 'puppet00', -# ports => '8140', -# server_names => ['server01', 'server02'], -# ipaddresses => ['192.168.56.200', '192.168.56.201'], -# options => 'check', -# } -# -# (this resource can be declared anywhere) -# -define haproxy::balancermember ( - $listening_service, - $ports, - $server_names = $::hostname, - $ipaddresses = $::ipaddress, - $options = '' -) { - # Template uses $ipaddresses, $server_name, $ports, $option - concat::fragment { "${listening_service}_balancermember_${name}": - order => "20-${listening_service}-${name}", - target => '/etc/haproxy/haproxy.cfg', - content => template('haproxy/haproxy_balancermember.erb'), - } -} diff --git a/puppet/modules/haproxy/manifests/init.pp b/puppet/modules/haproxy/manifests/init.pp deleted file mode 100644 index b91591a3..00000000 --- a/puppet/modules/haproxy/manifests/init.pp +++ /dev/null @@ -1,149 +0,0 @@ -# == Class: haproxy -# -# A Puppet module, using storeconfigs, to model an haproxy configuration. -# Currently VERY limited - Pull requests accepted! -# -# === Requirement/Dependencies: -# -# Currently requires the ripienaar/concat module on the Puppet Forge and -# uses storeconfigs on the Puppet Master to export/collect resources -# from all balancer members. -# -# === Parameters -# -# [*enable*] -# Chooses whether haproxy should be installed or ensured absent. -# Currently ONLY accepts valid boolean true/false values. -# -# [*version*] -# Allows you to specify what version of the package to install. -# Default is simply 'present' -# -# [*global_options*] -# A hash of all the haproxy global options. If you want to specify more -# than one option (i.e. multiple timeout or stats options), pass those -# options as an array and you will get a line for each of them in the -# resultant haproxy.cfg file. -# -# [*defaults_options*] -# A hash of all the haproxy defaults options. If you want to specify more -# than one option (i.e. multiple timeout or stats options), pass those -# options as an array and you will get a line for each of them in the -# resultant haproxy.cfg file. -# -# -# === Examples -# -# class { 'haproxy': -# enable => true, -# global_options => { -# 'log' => "${::ipaddress} local0", -# 'chroot' => '/var/lib/haproxy', -# 'pidfile' => '/var/run/haproxy.pid', -# 'maxconn' => '4000', -# 'user' => 'haproxy', -# 'group' => 'haproxy', -# 'daemon' => '', -# 'stats' => 'socket /var/lib/haproxy/stats' -# }, -# defaults_options => { -# 'log' => 'global', -# 'stats' => 'enable', -# 'option' => 'redispatch', -# 'retries' => '3', -# 'timeout' => [ -# 'http-request 10s', -# 'queue 1m', -# 'connect 10s', -# 'client 1m', -# 'server 1m', -# 'check 10s' -# ], -# 'maxconn' => '8000' -# }, -# } -# -class haproxy ( - $manage_service = true, - $enable = true, - $version = 'present', - $global_options = $haproxy::params::global_options, - $defaults_options = $haproxy::params::defaults_options -) inherits haproxy::params { - include concat::setup - - package { 'haproxy': - ensure => $enable ? { - true => $version, - false => absent, - }, - name => 'haproxy', - } - - if $enable { - concat { '/etc/haproxy/haproxy.cfg': - owner => '0', - group => '0', - mode => '0644', - require => Package['haproxy'], - notify => $manage_service ? { - true => Service['haproxy'], - false => undef, - }, - } - - # Simple Header - concat::fragment { '00-header': - target => '/etc/haproxy/haproxy.cfg', - order => '01', - content => "# This file managed by Puppet\n", - } - - # Template uses $global_options, $defaults_options - concat::fragment { 'haproxy-base': - target => '/etc/haproxy/haproxy.cfg', - order => '10', - content => template('haproxy/haproxy-base.cfg.erb'), - } - - if ($::osfamily == 'Debian') { - file { '/etc/default/haproxy': - content => 'ENABLED=1', - require => Package['haproxy'], - before => $manage_service ? { - true => Service['haproxy'], - false => undef, - }, - } - } - - file { $global_options['chroot']: - ensure => directory, - owner => $global_options['user'], - group => $global_options['group'], - mode => '0550', - require => Package['haproxy'] - } - - } - - if $manage_service { - service { 'haproxy': - ensure => $enable ? { - true => running, - false => stopped, - }, - enable => $enable ? { - true => true, - false => false, - }, - name => 'haproxy', - hasrestart => true, - hasstatus => true, - require => [ - Concat['/etc/haproxy/haproxy.cfg'], - File[$global_options['chroot']], - ], - } - } -} diff --git a/puppet/modules/haproxy/manifests/listen.pp b/puppet/modules/haproxy/manifests/listen.pp deleted file mode 100644 index 00636e3d..00000000 --- a/puppet/modules/haproxy/manifests/listen.pp +++ /dev/null @@ -1,95 +0,0 @@ -# == Define Resource Type: haproxy::listen -# -# This type will setup a listening service configuration block inside -# the haproxy.cfg file on an haproxy load balancer. Each listening service -# configuration needs one or more load balancer member server (that can be -# declared with the haproxy::balancermember defined resource type). Using -# storeconfigs, you can export the haproxy::balancermember resources on all -# load balancer member servers, and then collect them on a single haproxy -# load balancer server. -# -# === Requirement/Dependencies: -# -# Currently requires the ripienaar/concat module on the Puppet Forge and -# uses storeconfigs on the Puppet Master to export/collect resources -# from all balancer members. -# -# === Parameters -# -# [*name*] -# The namevar of the defined resource type is the listening service's name. -# This name goes right after the 'listen' statement in haproxy.cfg -# -# [*ports*] -# Ports on which the proxy will listen for connections on the ip address -# specified in the virtual_ip parameter. Accepts either a single -# comma-separated string or an array of strings which may be ports or -# hyphenated port ranges. -# -# [*ipaddress*] -# The ip address the proxy binds to. Empty addresses, '*', and '0.0.0.0' -# mean that the proxy listens to all valid addresses on the system. -# -# [*mode*] -# The mode of operation for the listening service. Valid values are 'tcp', -# HTTP', and 'health'. -# -# [*options*] -# A hash of options that are inserted into the listening service -# configuration block. -# -# [*collect_exported*] -# Boolean, default 'true'. True means 'collect exported @@balancermember resources' -# (for the case when every balancermember node exports itself), false means -# 'rely on the existing declared balancermember resources' (for the case when you -# know the full set of balancermembers in advance and use haproxy::balancermember -# with array arguments, which allows you to deploy everything in 1 run) -# -# -# === Examples -# -# Exporting the resource for a balancer member: -# -# haproxy::listen { 'puppet00': -# ipaddress => $::ipaddress, -# ports => '18140', -# mode => 'tcp', -# options => { -# 'option' => [ -# 'tcplog', -# 'ssl-hello-chk' -# ], -# 'balance' => 'roundrobin' -# }, -# } -# -# === Authors -# -# Gary Larizza <gary@puppetlabs.com> -# -define haproxy::listen ( - $ports, - $ipaddress = [$::ipaddress], - $mode = 'tcp', - $collect_exported = true, - $options = { - 'option' => [ - 'tcplog', - 'ssl-hello-chk' - ], - 'balance' => 'roundrobin' - } -) { - # Template uses: $name, $ipaddress, $ports, $options - concat::fragment { "${name}_listen_block": - order => "20-${name}-00", - target => '/etc/haproxy/haproxy.cfg', - content => template('haproxy/haproxy_listen_block.erb'), - } - - if $collect_exported { - Haproxy::Balancermember <<| listening_service == $name |>> - } - # else: the resources have been created and they introduced their - # concat fragments. We don't have to do anything about them. -} diff --git a/puppet/modules/haproxy/manifests/params.pp b/puppet/modules/haproxy/manifests/params.pp deleted file mode 100644 index 53442ddc..00000000 --- a/puppet/modules/haproxy/manifests/params.pp +++ /dev/null @@ -1,65 +0,0 @@ -# == Class: haproxy::params -# -# This is a container class holding default parameters for for haproxy class. -# currently, only the Redhat family is supported, but this can be easily -# extended by changing package names and configuration file paths. -# -class haproxy::params { - case $osfamily { - Redhat: { - $global_options = { - 'log' => "${::ipaddress} local0", - 'chroot' => '/var/lib/haproxy', - 'pidfile' => '/var/run/haproxy.pid', - 'maxconn' => '4000', - 'user' => 'haproxy', - 'group' => 'haproxy', - 'daemon' => '', - 'stats' => 'socket /var/lib/haproxy/stats' - } - $defaults_options = { - 'log' => 'global', - 'stats' => 'enable', - 'option' => 'redispatch', - 'retries' => '3', - 'timeout' => [ - 'http-request 10s', - 'queue 1m', - 'connect 10s', - 'client 1m', - 'server 1m', - 'check 10s', - ], - 'maxconn' => '8000' - } - } - Debian: { - $global_options = { - 'log' => "${::ipaddress} local0", - 'chroot' => '/var/lib/haproxy', - 'pidfile' => '/var/run/haproxy.pid', - 'maxconn' => '4000', - 'user' => 'haproxy', - 'group' => 'haproxy', - 'daemon' => '', - 'stats' => 'socket /var/lib/haproxy/stats' - } - $defaults_options = { - 'log' => 'global', - 'stats' => 'enable', - 'option' => 'redispatch', - 'retries' => '3', - 'timeout' => [ - 'http-request 10s', - 'queue 1m', - 'connect 10s', - 'client 1m', - 'server 1m', - 'check 10s', - ], - 'maxconn' => '8000' - } - } - default: { fail("The $::osfamily operating system is not supported with the haproxy module") } - } -} diff --git a/puppet/modules/haproxy/spec/classes/haproxy_spec.rb b/puppet/modules/haproxy/spec/classes/haproxy_spec.rb deleted file mode 100644 index 4b5902ce..00000000 --- a/puppet/modules/haproxy/spec/classes/haproxy_spec.rb +++ /dev/null @@ -1,138 +0,0 @@ -require 'spec_helper' - -describe 'haproxy', :type => :class do - let(:default_facts) do - { - :concat_basedir => '/dne', - :ipaddress => '10.10.10.10' - } - end - context 'on supported platforms' do - describe 'for OS-agnostic configuration' do - ['Debian', 'RedHat'].each do |osfamily| - context "on #{osfamily} family operatingsystems" do - let(:facts) do - { :osfamily => osfamily }.merge default_facts - end - let(:params) do - {'enable' => true} - end - it { should include_class('concat::setup') } - it 'should install the haproxy package' do - subject.should contain_package('haproxy').with( - 'ensure' => 'present' - ) - end - it 'should install the haproxy service' do - subject.should contain_service('haproxy').with( - 'ensure' => 'running', - 'enable' => 'true', - 'hasrestart' => 'true', - 'hasstatus' => 'true', - 'require' => [ - 'Concat[/etc/haproxy/haproxy.cfg]', - 'File[/var/lib/haproxy]' - ] - ) - end - it 'should set up /etc/haproxy/haproxy.cfg as a concat resource' do - subject.should contain_concat('/etc/haproxy/haproxy.cfg').with( - 'owner' => '0', - 'group' => '0', - 'mode' => '0644' - ) - end - it 'should manage the chroot directory' do - subject.should contain_file('/var/lib/haproxy').with( - 'ensure' => 'directory' - ) - end - it 'should contain a header concat fragment' do - subject.should contain_concat__fragment('00-header').with( - 'target' => '/etc/haproxy/haproxy.cfg', - 'order' => '01', - 'content' => "# This file managed by Puppet\n" - ) - end - it 'should contain a haproxy-base concat fragment' do - subject.should contain_concat__fragment('haproxy-base').with( - 'target' => '/etc/haproxy/haproxy.cfg', - 'order' => '10' - ) - end - describe 'Base concat fragment contents' do - let(:contents) { param_value(subject, 'concat::fragment', 'haproxy-base', 'content').split("\n") } - it 'should contain global and defaults sections' do - contents.should include('global') - contents.should include('defaults') - end - it 'should log to an ip address for local0' do - contents.should be_any { |match| match =~ / log \d+(\.\d+){3} local0/ } - end - it 'should specify the default chroot' do - contents.should include(' chroot /var/lib/haproxy') - end - it 'should specify the correct user' do - contents.should include(' user haproxy') - end - it 'should specify the correct group' do - contents.should include(' group haproxy') - end - it 'should specify the correct pidfile' do - contents.should include(' pidfile /var/run/haproxy.pid') - end - end - end - context "on #{osfamily} family operatingsystems without managing the service" do - let(:facts) do - { :osfamily => osfamily }.merge default_facts - end - let(:params) do - { - 'enable' => true, - 'manage_service' => false, - } - end - it { should include_class('concat::setup') } - it 'should install the haproxy package' do - subject.should contain_package('haproxy').with( - 'ensure' => 'present' - ) - end - it 'should install the haproxy service' do - subject.should_not contain_service('haproxy') - end - end - end - end - describe 'for OS-specific configuration' do - context 'only on Debian family operatingsystems' do - let(:facts) do - { :osfamily => 'Debian' }.merge default_facts - end - it 'should manage haproxy service defaults' do - subject.should contain_file('/etc/default/haproxy').with( - 'before' => 'Service[haproxy]', - 'require' => 'Package[haproxy]' - ) - verify_contents(subject, '/etc/default/haproxy', ['ENABLED=1']) - end - end - context 'only on RedHat family operatingsystems' do - let(:facts) do - { :osfamily => 'RedHat' }.merge default_facts - end - end - end - end - context 'on unsupported operatingsystems' do - let(:facts) do - { :osfamily => 'RainbowUnicorn' }.merge default_facts - end - it do - expect { - should contain_service('haproxy') - }.to raise_error(Puppet::Error, /operating system is not supported with the haproxy module/) - end - end -end diff --git a/puppet/modules/haproxy/spec/defines/balancermember_spec.rb b/puppet/modules/haproxy/spec/defines/balancermember_spec.rb deleted file mode 100644 index 74bc7a8b..00000000 --- a/puppet/modules/haproxy/spec/defines/balancermember_spec.rb +++ /dev/null @@ -1,82 +0,0 @@ -require 'spec_helper' - -describe 'haproxy::balancermember' do - let(:title) { 'tyler' } - let(:facts) do - { - :ipaddress => '1.1.1.1', - :hostname => 'dero' - } - end - - context 'with a single balancermember option' do - let(:params) do - { - :name => 'tyler', - :listening_service => 'croy', - :ports => '18140', - :options => 'check' - } - end - - it { should contain_concat__fragment('croy_balancermember_tyler').with( - 'order' => '20-croy-tyler', - 'target' => '/etc/haproxy/haproxy.cfg', - 'content' => " server dero 1.1.1.1:18140 check\n\n" - ) } - end - - context 'with multiple balancermember options' do - let(:params) do - { - :name => 'tyler', - :listening_service => 'croy', - :ports => '18140', - :options => ['check', 'close'] - } - end - - it { should contain_concat__fragment('croy_balancermember_tyler').with( - 'order' => '20-croy-tyler', - 'target' => '/etc/haproxy/haproxy.cfg', - 'content' => " server dero 1.1.1.1:18140 check close\n\n" - ) } - end - - context 'with multiple servers' do - let(:params) do - { - :name => 'tyler', - :listening_service => 'croy', - :ports => '18140', - :server_names => ['server01', 'server02'], - :ipaddresses => ['192.168.56.200', '192.168.56.201'], - :options => ['check'] - } - end - - it { should contain_concat__fragment('croy_balancermember_tyler').with( - 'order' => '20-croy-tyler', - 'target' => '/etc/haproxy/haproxy.cfg', - 'content' => " server server01 192.168.56.200:18140 check\n server server02 192.168.56.201:18140 check\n\n" - ) } - end - context 'with multiple servers and multiple ports' do - let(:params) do - { - :name => 'tyler', - :listening_service => 'croy', - :ports => ['18140','18150'], - :server_names => ['server01', 'server02'], - :ipaddresses => ['192.168.56.200', '192.168.56.201'], - :options => ['check'] - } - end - - it { should contain_concat__fragment('croy_balancermember_tyler').with( - 'order' => '20-croy-tyler', - 'target' => '/etc/haproxy/haproxy.cfg', - 'content' => " server server01 192.168.56.200:18140,192.168.56.200:18150 check\n server server02 192.168.56.201:18140,192.168.56.201:18150 check\n\n" - ) } - end -end diff --git a/puppet/modules/haproxy/spec/defines/listen_spec.rb b/puppet/modules/haproxy/spec/defines/listen_spec.rb deleted file mode 100644 index 31dd4c85..00000000 --- a/puppet/modules/haproxy/spec/defines/listen_spec.rb +++ /dev/null @@ -1,53 +0,0 @@ -require 'spec_helper' - -describe 'haproxy::listen' do - let(:title) { 'tyler' } - let(:facts) {{ :ipaddress => '1.1.1.1' }} - context "when only one port is provided" do - let(:params) do - { - :name => 'croy', - :ports => '18140' - } - end - - it { should contain_concat__fragment('croy_listen_block').with( - 'order' => '20-croy-00', - 'target' => '/etc/haproxy/haproxy.cfg', - 'content' => "listen croy\n\n bind 1.1.1.1:18140\n\n balance roundrobin\n option tcplog\n option ssl-hello-chk\n" - ) } - end - context "when an array of ports is provided" do - let(:params) do - { - :name => 'apache', - :ipaddress => '23.23.23.23', - :ports => [ - '80', - '443', - ] - } - end - - it { should contain_concat__fragment('apache_listen_block').with( - 'order' => '20-apache-00', - 'target' => '/etc/haproxy/haproxy.cfg', - 'content' => "listen apache\n\n bind 23.23.23.23:80\n\n bind 23.23.23.23:443\n\n balance roundrobin\n option tcplog\n option ssl-hello-chk\n" - ) } - end - context "when a comma-separated list of ports is provided" do - let(:params) do - { - :name => 'apache', - :ipaddress => '23.23.23.23', - :ports => '80,443' - } - end - - it { should contain_concat__fragment('apache_listen_block').with( - 'order' => '20-apache-00', - 'target' => '/etc/haproxy/haproxy.cfg', - 'content' => "listen apache\n\n bind 23.23.23.23:80\n\n bind 23.23.23.23:443\n\n balance roundrobin\n option tcplog\n option ssl-hello-chk\n" - ) } - end -end diff --git a/puppet/modules/haproxy/spec/spec.opts b/puppet/modules/haproxy/spec/spec.opts deleted file mode 100644 index 91cd6427..00000000 --- a/puppet/modules/haproxy/spec/spec.opts +++ /dev/null @@ -1,6 +0,0 @@ ---format -s ---colour ---loadby -mtime ---backtrace diff --git a/puppet/modules/haproxy/spec/spec_helper.rb b/puppet/modules/haproxy/spec/spec_helper.rb deleted file mode 100644 index 2c6f5664..00000000 --- a/puppet/modules/haproxy/spec/spec_helper.rb +++ /dev/null @@ -1 +0,0 @@ -require 'puppetlabs_spec_helper/module_spec_helper' diff --git a/puppet/modules/haproxy/templates/haproxy-base.cfg.erb b/puppet/modules/haproxy/templates/haproxy-base.cfg.erb deleted file mode 100644 index f25d5c34..00000000 --- a/puppet/modules/haproxy/templates/haproxy-base.cfg.erb +++ /dev/null @@ -1,21 +0,0 @@ -global -<% @global_options.sort.each do |key,val| -%> -<% if val.is_a?(Array) -%> -<% val.each do |item| -%> - <%= key %> <%= item %> -<% end -%> -<% else -%> - <%= key %> <%= val %> -<% end -%> -<% end -%> - -defaults -<% @defaults_options.sort.each do |key,val| -%> -<% if val.is_a?(Array) -%> -<% val.each do |item| -%> - <%= key %> <%= item %> -<% end -%> -<% else -%> - <%= key %> <%= val %> -<% end -%> -<% end -%> diff --git a/puppet/modules/haproxy/templates/haproxy_balancermember.erb b/puppet/modules/haproxy/templates/haproxy_balancermember.erb deleted file mode 100644 index 1d03f565..00000000 --- a/puppet/modules/haproxy/templates/haproxy_balancermember.erb +++ /dev/null @@ -1,3 +0,0 @@ -<% Array(ipaddresses).zip(Array(server_names)).each do |ipaddress,host| -%> - server <%= host %> <%= ipaddress %>:<%= Array(ports).collect {|x|x.split(',')}.flatten.join(",#{ipaddress}:") %> <%= Array(options).join(" ") %> -<% end %> diff --git a/puppet/modules/haproxy/templates/haproxy_listen_block.erb b/puppet/modules/haproxy/templates/haproxy_listen_block.erb deleted file mode 100644 index 129313f1..00000000 --- a/puppet/modules/haproxy/templates/haproxy_listen_block.erb +++ /dev/null @@ -1,10 +0,0 @@ -listen <%= name %> - mode <%= mode %> -<% Array(ipaddress).uniq.each do |virtual_ip| (ports.is_a?(Array) ? ports : Array(ports.split(","))).each do |port| %> - bind <%= virtual_ip %>:<%= port %> -<% end end %> -<% options.sort.each do |key, val| -%> -<% Array(val).each do |item| -%> - <%= key %> <%= item %> -<% end -%> -<% end -%> diff --git a/puppet/modules/haproxy/tests/init.pp b/puppet/modules/haproxy/tests/init.pp deleted file mode 100644 index 77590ac8..00000000 --- a/puppet/modules/haproxy/tests/init.pp +++ /dev/null @@ -1,69 +0,0 @@ -# Declare haproxy base class with configuration options -class { 'haproxy': - enable => true, - global_options => { - 'log' => "${::ipaddress} local0", - 'chroot' => '/var/lib/haproxy', - 'pidfile' => '/var/run/haproxy.pid', - 'maxconn' => '4000', - 'user' => 'haproxy', - 'group' => 'haproxy', - 'daemon' => '', - 'stats' => 'socket /var/lib/haproxy/stats', - }, - defaults_options => { - 'log' => 'global', - 'stats' => 'enable', - 'option' => 'redispatch', - 'retries' => '3', - 'timeout' => [ - 'http-request 10s', - 'queue 1m', - 'connect 10s', - 'client 1m', - 'server 1m', - 'check 10s', - ], - 'maxconn' => '8000', - }, -} - -# Export a balancermember server, note that the listening_service parameter -# will/must correlate with an haproxy::listen defined resource type. -@@haproxy::balancermember { $fqdn: - order => '21', - listening_service => 'puppet00', - server_name => $::hostname, - balancer_ip => $::ipaddress, - balancer_port => '8140', - balancermember_options => 'check' -} - -# Declare a couple of Listening Services for haproxy.cfg -# Note that the balancermember server resources are being collected in -# the haproxy::config defined resource type with the following line: -# Haproxy::Balancermember <<| listening_service == $name |>> -haproxy::listen { 'puppet00': - order => '20', - ipaddress => $::ipaddress, - ports => '18140', - options => { - 'option' => [ - 'tcplog', - 'ssl-hello-chk', - ], - 'balance' => 'roundrobin', - }, -} -haproxy::listen { 'stats': - order => '30', - ipaddress => '', - ports => '9090', - options => { - 'mode' => 'http', - 'stats' => [ - 'uri /', - 'auth puppet:puppet' - ], - }, -} diff --git a/puppet/modules/leap/manifests/cli/install.pp b/puppet/modules/leap/manifests/cli/install.pp index 25e87033..d009316b 100644 --- a/puppet/modules/leap/manifests/cli/install.pp +++ b/puppet/modules/leap/manifests/cli/install.pp @@ -1,13 +1,20 @@ # installs leap_cli on node class leap::cli::install ( $source = false ) { + + # nokogiri is a dependency gem of leap_cli and + # needs build tools in order to get compiled + ensure_packages (['gcc', 'make', 'zlib1g-dev']) + class { '::ruby': + install_dev => true, + require => [ Package['gcc'], Package['make'], Package['zlib1g-dev'] ] + } + + if $source { # needed for building leap_cli from source include ::git include ::rubygems - class { '::ruby': - install_dev => true - } class { 'bundler::install': install_method => 'package' } @@ -40,7 +47,8 @@ class leap::cli::install ( $source = false ) { else { package { 'leap_cli': ensure => installed, - provider => gem + provider => gem, + require => Class['ruby'] } } } diff --git a/puppet/modules/leap_mx/manifests/init.pp b/puppet/modules/leap_mx/manifests/init.pp index d758e3ab..f26448e2 100644 --- a/puppet/modules/leap_mx/manifests/init.pp +++ b/puppet/modules/leap_mx/manifests/init.pp @@ -6,7 +6,7 @@ class leap_mx { $couchdb_password = $leap_mx['password'] $couchdb_host = 'localhost' - $couchdb_port = '4096' + $couchdb_port = hiera('couchdb_port') $sources = hiera('sources') @@ -94,15 +94,11 @@ class leap_mx { # LEAP-MX CODE AND DEPENDENCIES # - package { - $sources['leap-mx']['package']: - ensure => $sources['leap-mx']['revision'], - require => [ - Class['site_apt::leap_repo'], - User['leap-mx'] ]; - - 'leap-keymanager': - ensure => latest; + package { $sources['leap-mx']['package']: + ensure => $sources['leap-mx']['revision'], + require => [ + Class['site_apt::leap_repo'], + User['leap-mx'] ]; } # diff --git a/puppet/modules/site_apache/files/autorestart.conf b/puppet/modules/site_apache/files/autorestart.conf new file mode 100644 index 00000000..8a764e34 --- /dev/null +++ b/puppet/modules/site_apache/files/autorestart.conf @@ -0,0 +1,2 @@ +[Service] +Restart=always diff --git a/puppet/modules/site_apache/manifests/common.pp b/puppet/modules/site_apache/manifests/common.pp index 208c15d5..74116575 100644 --- a/puppet/modules/site_apache/manifests/common.pp +++ b/puppet/modules/site_apache/manifests/common.pp @@ -13,20 +13,13 @@ class site_apache::common { # needed for the mod_ssl config include apache::module::mime - # load mods depending on apache version - if ( $::lsbdistcodename == 'jessie' ) { - # apache >= 2.4, debian jessie - # needed for mod_ssl config - include apache::module::socache_shmcb - # generally needed - include apache::module::mpm_prefork - } else { - # apache < 2.4, debian wheezy - # for "Order" directive, i.e. main apache2.conf - include apache::module::authz_host - } + # needed for mod_ssl config + include apache::module::socache_shmcb + # generally needed + include apache::module::mpm_prefork include site_apache::common::tls include site_apache::common::acme + include site_apache::common::autorestart } diff --git a/puppet/modules/site_apache/manifests/common/autorestart.pp b/puppet/modules/site_apache/manifests/common/autorestart.pp new file mode 100644 index 00000000..8b7b590d --- /dev/null +++ b/puppet/modules/site_apache/manifests/common/autorestart.pp @@ -0,0 +1,30 @@ +# +# Adds autorestart extension to apache on crash +# +class site_apache::common::autorestart { + + file { + '/etc/systemd/system/apache2.service.d': + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755'; + + # Add .placeholder file so directory doesn't get removed by + # deb-systemd-helper in a package removal postrm, see + # issue #8841 for more details. + '/etc/systemd/system/apache2.service.d/.placeholder': + ensure => file, + owner => 'root', + group => 'root', + mode => '0755'; + } + + ::systemd::unit_file { 'apache2.service.d/autorestart.conf': + source => 'puppet:///modules/site_apache/autorestart.conf', + require => [ + File['/etc/systemd/system/apache2.service.d'], + Service['apache'], + ] + } +} diff --git a/puppet/modules/site_apache/spec/classes/autorestart_spec.rb b/puppet/modules/site_apache/spec/classes/autorestart_spec.rb new file mode 100644 index 00000000..ad9c9f2e --- /dev/null +++ b/puppet/modules/site_apache/spec/classes/autorestart_spec.rb @@ -0,0 +1,7 @@ +require 'spec_helper' + +describe 'site_apache::common::autorestart' do + it "should include apache autorestart" do + should contain_file('apache2.service.d/autorestart.conf').with_source('puppet:///modules/site_apache/autorestart.conf') + end +end diff --git a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb index 1d19094e..ddf69a42 100644 --- a/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb +++ b/puppet/modules/site_apache/templates/vhosts.d/hidden_service.conf.erb @@ -1,5 +1,5 @@ <VirtualHost 127.0.0.1:80> - ServerName <%= @tor_domain %> + ServerName <%= @onion_domain %> <IfModule mod_headers.c> Header always unset X-Powered-By diff --git a/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap b/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap deleted file mode 100644 index bbaac6a2..00000000 --- a/puppet/modules/site_apt/files/Debian/51unattended-upgrades-leap +++ /dev/null @@ -1,6 +0,0 @@ -// this file is managed by puppet ! - -Unattended-Upgrade::Allowed-Origins { - "leap.se:stable"; -} - diff --git a/puppet/modules/site_apt/manifests/init.pp b/puppet/modules/site_apt/manifests/init.pp index 26bd2c6a..60fe0483 100644 --- a/puppet/modules/site_apt/manifests/init.pp +++ b/puppet/modules/site_apt/manifests/init.pp @@ -12,21 +12,32 @@ class site_apt { # leap repo url $platform_sources = $sources['platform'] $apt_url_platform_basic = $platform_sources['apt']['basic'] + $apt_platform_component = $platform_sources['apt']['component'] - # needed on jessie hosts for getting pnp4nagios from testing + if ( $platform_sources['apt']['codename'] == '') { + $apt_platform_codename = $::lsbdistcodename + } else { + $apt_platform_codename = $platform_sources['apt']['codename'] + } + + # needed on jessie hosts for getting python-treq from stretch + # see https://0xacab.org/leap/platform/issues/8836 if ( $::operatingsystemmajrelease == '8' ) { - $use_next_release = true + $use_next_release = true + $custom_preferences = template("site_apt/${::operatingsystem}/preferences_jessie.erb") } else { - $use_next_release = false + $use_next_release = false + $custom_preferences = '' } class { 'apt': - custom_key_dir => 'puppet:///modules/site_apt/keys', - debian_url => $apt_url_basic, - security_url => $apt_url_security, - backports_url => $apt_url_backports, - use_next_release => $use_next_release, - repos => 'main' + custom_key_dir => 'puppet:///modules/site_apt/keys', + debian_url => $apt_url_basic, + security_url => $apt_url_security, + backports_url => $apt_url_backports, + use_next_release => $use_next_release, + custom_preferences => $custom_preferences, + repos => 'main' } # enable http://deb.leap.se debian package repository diff --git a/puppet/modules/site_apt/manifests/leap_repo.pp b/puppet/modules/site_apt/manifests/leap_repo.pp index 5eedce45..1e18b441 100644 --- a/puppet/modules/site_apt/manifests/leap_repo.pp +++ b/puppet/modules/site_apt/manifests/leap_repo.pp @@ -4,8 +4,25 @@ class site_apt::leap_repo { $platform = hiera_hash('platform') $major_version = $platform['major_version'] + # on jessie, keys need to be in /etc/apt/... + # see https://0xacab.org/leap/platform/issues/8862 + if ( $::operatingsystemmajrelease == '8' ) { + if $::site_apt::apt_platform_component =~ /.*(staging|master).*/ { + $archive_key = 'CE433F407BAB443AFEA196C1837C1AD5367429D9' + } else { + $archive_key = '1E453B2CE87BEE2F7DFE99661E34A1828E207901' + } + } + if ( $::operatingsystemmajrelease != '8' ) { + if $::site_apt::apt_platform_component =~ /.*(staging|master).*/ { + $archive_key = '/usr/share/keyrings/leap-experimental-archive.gpg' + } else { + $archive_key = '/usr/share/keyrings/leap-archive.gpg' + } + } + apt::sources_list { 'leap.list': - content => "deb ${::site_apt::apt_url_platform_basic} ${::lsbdistcodename} main\n", + content => "deb [signed-by=${archive_key}] ${::site_apt::apt_url_platform_basic} ${::site_apt::apt_platform_component} ${::site_apt::apt_platform_codename}\n", before => Exec[refresh_apt] } diff --git a/puppet/modules/site_apt/manifests/preferences/passenger.pp b/puppet/modules/site_apt/manifests/preferences/passenger.pp deleted file mode 100644 index 8cd41f91..00000000 --- a/puppet/modules/site_apt/manifests/preferences/passenger.pp +++ /dev/null @@ -1,14 +0,0 @@ -# -# currently, this is only used by static_site to get passenger v4. -# -# UPGRADE: this is not needed for jessie. -# -class site_apt::preferences::passenger { - - apt::preferences_snippet { 'passenger': - package => 'libapache2-mod-passenger', - release => "${::lsbdistcodename}-backports", - priority => 999; - } - -} diff --git a/puppet/modules/site_apt/manifests/preferences/python_cryptography.pp b/puppet/modules/site_apt/manifests/preferences/python_cryptography.pp new file mode 100644 index 00000000..d725c1af --- /dev/null +++ b/puppet/modules/site_apt/manifests/preferences/python_cryptography.pp @@ -0,0 +1,12 @@ +# Pin python-cryptography to jessie-backports in order to +# satisfy leap-mx dependency (>=17.0) +# see https://0xacab.org/leap/platform/issues/8837 +class site_apt::preferences::python_cryptography { + + apt::preferences_snippet { 'python_cryptography': + package => 'python-cryptography python-openssl python-pyasn1 python-setuptools python-pkg-resources python-cffi', + release => "${::lsbdistcodename}-backports", + priority => 999; + } + +} diff --git a/puppet/modules/site_apt/manifests/preferences/rsyslog.pp b/puppet/modules/site_apt/manifests/preferences/rsyslog.pp deleted file mode 100644 index bfeaa7da..00000000 --- a/puppet/modules/site_apt/manifests/preferences/rsyslog.pp +++ /dev/null @@ -1,13 +0,0 @@ -class site_apt::preferences::rsyslog { - - apt::preferences_snippet { - 'rsyslog_anon_depends': - package => 'libestr0 librelp0 rsyslog*', - priority => '999', - pin => 'release a=wheezy-backports', - before => Class['rsyslog::install']; - - 'fixed_rsyslog_anon_package': - ensure => absent; - } -} diff --git a/puppet/modules/site_apt/manifests/unattended_upgrades.pp b/puppet/modules/site_apt/manifests/unattended_upgrades.pp index 42f1f4c6..ddadd35a 100644 --- a/puppet/modules/site_apt/manifests/unattended_upgrades.pp +++ b/puppet/modules/site_apt/manifests/unattended_upgrades.pp @@ -11,8 +11,7 @@ class site_apt::unattended_upgrades { # configure LEAP upgrades apt::apt_conf { '51unattended-upgrades-leap': - source => [ - "puppet:///modules/site_apt/${::lsbdistid}/51unattended-upgrades-leap"], + content => template('site_apt/51unattended-upgrades-leap'), require => Package['unattended-upgrades'], refresh_apt => false, } diff --git a/puppet/modules/site_apt/templates/51unattended-upgrades-leap b/puppet/modules/site_apt/templates/51unattended-upgrades-leap new file mode 100644 index 00000000..2a3494ef --- /dev/null +++ b/puppet/modules/site_apt/templates/51unattended-upgrades-leap @@ -0,0 +1,5 @@ +// this file is managed by puppet ! + +Unattended-Upgrade::Origins-Pattern { + "site=deb.leap.se,codename=<%= @apt_platform_component %>"; +} diff --git a/puppet/modules/site_apt/templates/Debian/preferences_jessie.erb b/puppet/modules/site_apt/templates/Debian/preferences_jessie.erb new file mode 100644 index 00000000..879885dd --- /dev/null +++ b/puppet/modules/site_apt/templates/Debian/preferences_jessie.erb @@ -0,0 +1,19 @@ +Explanation: Debian jessie +Package: * +Pin: release o=Debian,n=jessie +Pin-Priority: 990 + +Explanation: Debian stretch +Package: * +Pin: release o=Debian,n=stretch +Pin-Priority: 1 + +Explanation: Debian sid +Package: * +Pin: release o=Debian,n=sid +Pin-Priority: 1 + +Explanation: Debian fallback +Package: * +Pin: release o=Debian +Pin-Priority: -10 diff --git a/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg b/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg index 337d9ec6..a5375cc8 100644 --- a/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg +++ b/puppet/modules/site_check_mk/files/agent/logwatch/webapp.cfg @@ -1,6 +1,10 @@ /var/log/leap/webapp.log # check for webapp errors C Completed 500 +# also alert conflicts. They might be meaningful response codes +# but so far we were just handing them on from couch and they +# indicated some actual problem. + C Completed 409 # couch connection issues C webapp.*Could not connect to couch database messages due to 401 Unauthorized: {"error":"unauthorized","reason":"You are not a server admin."} # ignore RoutingErrors that rails throw when it can't handle a url diff --git a/puppet/modules/site_check_mk/files/extra_service_conf.mk b/puppet/modules/site_check_mk/files/extra_service_conf.mk index c7120a96..9212af95 100644 --- a/puppet/modules/site_check_mk/files/extra_service_conf.mk +++ b/puppet/modules/site_check_mk/files/extra_service_conf.mk @@ -1,6 +1,9 @@ # retry 3 times before setting a service into a hard state -# and send out notification +# Delay a hard state of the APT check for 1 day +# so unattended_upgrades has time to upgrade packages. +# extra_service_conf["max_check_attempts"] = [ + ("360", ALL_HOSTS , ["APT"] ), ("4", ALL_HOSTS , ALL_SERVICES ) ] @@ -11,4 +14,3 @@ extra_service_conf["max_check_attempts"] = [ extra_service_conf["normal_check_interval"] = [ ("4", ALL_HOSTS , "Check_MK" ) ] - diff --git a/puppet/modules/site_check_mk/manifests/agent/haproxy.pp b/puppet/modules/site_check_mk/manifests/agent/haproxy.pp deleted file mode 100644 index 6d52efba..00000000 --- a/puppet/modules/site_check_mk/manifests/agent/haproxy.pp +++ /dev/null @@ -1,15 +0,0 @@ -class site_check_mk::agent::haproxy { - - include site_check_mk::agent::package::nagios_plugins_contrib - - # local nagios plugin checks via mrpe - augeas { 'haproxy': - incl => '/etc/check_mk/mrpe.cfg', - lens => 'Spacevars.lns', - changes => [ - 'rm /files/etc/check_mk/mrpe.cfg/Haproxy', - 'set Haproxy \'/usr/lib/nagios/plugins/check_haproxy -u "http://localhost:8000/haproxy;csv"\'' ], - require => File['/etc/check_mk/mrpe.cfg']; - } - -} diff --git a/puppet/modules/site_config/lib/facter/vagrant.rb b/puppet/modules/site_config/lib/facter/vagrant.rb new file mode 100644 index 00000000..29a218dd --- /dev/null +++ b/puppet/modules/site_config/lib/facter/vagrant.rb @@ -0,0 +1,8 @@ +# Checks if systems runs inside vagrant +require 'facter' + +Facter.add(:vagrant) do + setcode do + FileTest.exists?('/vagrant') + end +end diff --git a/puppet/modules/site_config/manifests/files.pp b/puppet/modules/site_config/manifests/files.pp index d2ef8a98..e74ad567 100644 --- a/puppet/modules/site_config/manifests/files.pp +++ b/puppet/modules/site_config/manifests/files.pp @@ -3,10 +3,10 @@ class site_config::files { file { '/srv/leap': - ensure => directory, - owner => 'root', - group => 'root', - mode => '0711'; + ensure => directory, + owner => 'root', + group => 'root', + mode => '0711'; [ '/etc/leap', '/var/lib/leap']: ensure => directory, diff --git a/puppet/modules/site_config/manifests/packages/build_essential.pp b/puppet/modules/site_config/manifests/packages/build_essential.pp index 2b3e13b9..5b9a2602 100644 --- a/puppet/modules/site_config/manifests/packages/build_essential.pp +++ b/puppet/modules/site_config/manifests/packages/build_essential.pp @@ -16,12 +16,6 @@ class site_config::packages::build_essential inherits ::site_config::packages { } } - /^7.*/: { - Package[ 'gcc-4.7','g++-4.7', 'cpp-4.7' ] { - ensure => present - } - } - default: { } } diff --git a/puppet/modules/site_config/manifests/params.pp b/puppet/modules/site_config/manifests/params.pp index 012b3ce0..2c9687a3 100644 --- a/puppet/modules/site_config/manifests/params.pp +++ b/puppet/modules/site_config/manifests/params.pp @@ -1,3 +1,4 @@ +# Default parameters class site_config::params { $ip_address = hiera('ip_address') @@ -5,9 +6,16 @@ class site_config::params { $ec2_local_ipv4_interface = getvar("interface_${::ec2_local_ipv4}") $environment = hiera('environment', undef) - - if $environment == 'local' { - $interface = 'eth1' + if str2bool("$::vagrant") { + # Depending on the backend hypervisor networking is setup differently. + if $::interfaces =~ /eth1/ { + # Virtualbox: Private networking creates a second interface eth1 + $interface = 'eth1' + } + else { + # KVM/Libvirt: Private networking is done by defauly on first interface + $interface = 'eth0' + } include site_config::packages::build_essential } elsif hiera('interface','') != '' { diff --git a/puppet/modules/site_config/manifests/remove.pp b/puppet/modules/site_config/manifests/remove.pp index 443df9c2..be6cdfd8 100644 --- a/puppet/modules/site_config/manifests/remove.pp +++ b/puppet/modules/site_config/manifests/remove.pp @@ -2,6 +2,11 @@ class site_config::remove { include site_config::remove::files + package { 'leap-keyring': + ensure => purged, + } + + case $::operatingsystemrelease { /^8.*/: { include site_config::remove::jessie diff --git a/puppet/modules/site_config/manifests/remove/jessie.pp b/puppet/modules/site_config/manifests/remove/jessie.pp index e9497baf..2fdc4794 100644 --- a/puppet/modules/site_config/manifests/remove/jessie.pp +++ b/puppet/modules/site_config/manifests/remove/jessie.pp @@ -7,8 +7,9 @@ class site_config::remove::jessie { } apt::preferences_snippet { - [ 'facter', 'obfsproxy', 'python-twisted', 'unbound' ]: - ensure => absent; + [ 'facter', 'obfsproxy', 'python-twisted', 'unbound', 'passenger', + 'rsyslog_anon_depends' ]: + ensure => absent; } } diff --git a/puppet/modules/site_config/manifests/remove/webapp.pp b/puppet/modules/site_config/manifests/remove/webapp.pp index 58f59815..963eb705 100644 --- a/puppet/modules/site_config/manifests/remove/webapp.pp +++ b/puppet/modules/site_config/manifests/remove/webapp.pp @@ -4,4 +4,16 @@ class site_config::remove::webapp { '/etc/apache/sites-enabled/leap_webapp.conf': notify => Service['apache']; } + + # Ensure haproxy is removed + package { 'haproxy': + ensure => purged, + } + augeas { 'haproxy': + incl => '/etc/check_mk/mrpe.cfg', + lens => 'Spacevars.lns', + changes => [ 'rm /files/etc/check_mk/mrpe.cfg/Haproxy' ], + require => File['/etc/check_mk/mrpe.cfg']; + } + } diff --git a/puppet/modules/site_config/manifests/setup.pp b/puppet/modules/site_config/manifests/setup.pp index 82dfe76d..ce0f91d4 100644 --- a/puppet/modules/site_config/manifests/setup.pp +++ b/puppet/modules/site_config/manifests/setup.pp @@ -37,7 +37,7 @@ class site_config::setup { # we need to include shorewall::interface{eth0} in setup.pp so # packages can be installed during main puppetrun, even before shorewall # is configured completly - if ( $::site_config::params::environment == 'local' ) { + if str2bool($::vagrant) { include site_config::vagrant } diff --git a/puppet/modules/site_config/manifests/syslog.pp b/puppet/modules/site_config/manifests/syslog.pp index 591e0601..096d5d77 100644 --- a/puppet/modules/site_config/manifests/syslog.pp +++ b/puppet/modules/site_config/manifests/syslog.pp @@ -1,25 +1,17 @@ # configure rsyslog on all nodes class site_config::syslog { - # only pin rsyslog packages to backports on wheezy - case $::operatingsystemrelease { - /^7.*/: { - include ::site_apt::preferences::rsyslog - } - # on jessie+ systems, systemd and journald are enabled, - # and journald logs IP addresses, so we need to disable - # it until a solution is found, (#7863): - # https://github.com/systemd/systemd/issues/2447 - default: { - include ::journald - augeas { - 'disable_journald': - incl => '/etc/systemd/journald.conf', - lens => 'Puppet.lns', - changes => 'set /files/etc/systemd/journald.conf/Journal/Storage \'none\'', - notify => Service['systemd-journald']; - } - } + # on jessie+ systems, systemd and journald are enabled, + # and journald logs IP addresses, so we need to disable + # it until a solution is found, (#7863): + # https://github.com/systemd/systemd/issues/2447 + include ::journald + augeas { + 'disable_journald': + incl => '/etc/systemd/journald.conf', + lens => 'Puppet.lns', + changes => 'set /files/etc/systemd/journald.conf/Journal/Storage \'none\'', + notify => Service['systemd-journald']; } class { '::rsyslog::client': diff --git a/puppet/modules/site_config/manifests/vagrant.pp b/puppet/modules/site_config/manifests/vagrant.pp index 8f50b305..1682de8b 100644 --- a/puppet/modules/site_config/manifests/vagrant.pp +++ b/puppet/modules/site_config/manifests/vagrant.pp @@ -1,11 +1,15 @@ +# Gets included on vagrant nodes class site_config::vagrant { - # class for vagrant nodes include site_shorewall::defaults - # eth0 on vagrant nodes is the uplink if - shorewall::interface { 'eth0': - zone => 'net', - options => 'tcpflags,blacklist,nosmurfs'; + + if ( $::site_config::params::interface == 'eth1' ) { + # Don't block eth0 even if eth1 is configured, because + # it's vagrant's main interface to access the box + shorewall::interface { 'eth0': + zone => 'net', + options => 'tcpflags,blacklist,nosmurfs'; + } } } diff --git a/puppet/modules/site_couchdb/files/designs/identities/Identity.json b/puppet/modules/site_couchdb/files/designs/identities/Identity.json index b1c567c1..c099ae4a 100644 --- a/puppet/modules/site_couchdb/files/designs/identities/Identity.json +++ b/puppet/modules/site_couchdb/files/designs/identities/Identity.json @@ -9,14 +9,14 @@ "all": { "map": " function(doc) {\n if (doc['type'] == 'Identity') {\n emit(doc._id, null);\n }\n }\n" }, - "cert_fingerprints_by_expiry": { - "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.cert_fingerprints === \"object\") {\n for (fp in doc.cert_fingerprints) {\n if (doc.cert_fingerprints.hasOwnProperty(fp)) {\n emit(doc.cert_fingerprints[fp], fp);\n }\n }\n }\n}\n" + "disabled": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.user_id === \"undefined\") {\n emit(doc._id, 1);\n }\n}\n" }, "cert_expiry_by_fingerprint": { "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.cert_fingerprints === \"object\") {\n for (fp in doc.cert_fingerprints) {\n if (doc.cert_fingerprints.hasOwnProperty(fp)) {\n emit(fp, doc.cert_fingerprints[fp]);\n }\n }\n }\n}\n" }, - "disabled": { - "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.user_id === \"undefined\") {\n emit(doc._id, 1);\n }\n}\n" + "cert_fingerprints_by_expiry": { + "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.cert_fingerprints === \"object\") {\n for (fp in doc.cert_fingerprints) {\n if (doc.cert_fingerprints.hasOwnProperty(fp)) {\n emit(doc.cert_fingerprints[fp], fp);\n }\n }\n }\n}\n" }, "pgp_key_by_email": { "map": "function(doc) {\n if (doc.type != 'Identity') {\n return;\n }\n if (typeof doc.keys === \"object\") {\n emit(doc.address, doc.keys[\"pgp\"]);\n }\n}\n" diff --git a/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json index 006c1ea1..d6e1e9d5 100644 --- a/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json +++ b/puppet/modules/site_couchdb/files/designs/invite_codes/InviteCode.json @@ -1,22 +1,26 @@ { - "_id": "_design/InviteCode", - "language": "javascript", - "views": { - "by__id": { - "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['_id'] != null)) {\n emit(doc['_id'], 1);\n }\n }\n", - "reduce": "_sum" - }, - "by_invite_code": { - "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_code'] != null)) {\n emit(doc['invite_code'], 1);\n }\n }\n", - "reduce": "_sum" - }, - "by_invite_count": { - "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_count'] != null)) {\n emit(doc['invite_count'], 1);\n }\n }\n", - "reduce": "_sum" - }, - "all": { - "map": " function(doc) {\n if (doc['type'] == 'InviteCode') {\n emit(doc._id, null);\n }\n }\n" - } - }, - "couchrest-hash": "83fb8f504520b4a9c7ddbb7928cd0ce3" + "_id": "_design/InviteCode", + "language": "javascript", + "views": { + "by_invite_code": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_code'] != null)) {\n emit(doc['invite_code'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_invite_count": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['invite_count'] != null)) {\n emit(doc['invite_count'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_created_at": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['created_at'] != null)) {\n emit(doc['created_at'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "by_updated_at": { + "map": " function(doc) {\n if ((doc['type'] == 'InviteCode') && (doc['updated_at'] != null)) {\n emit(doc['updated_at'], 1);\n }\n }\n", + "reduce": "_sum" + }, + "all": { + "map": " function(doc) {\n if (doc['type'] == 'InviteCode') {\n emit(doc._id, null);\n }\n }\n" + } + }, + "couchrest-hash": "2d1883c83164a0be127c3a569d9c1902" }
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/messages/Message.json b/puppet/modules/site_couchdb/files/designs/messages/Message.json index 6a48fc4d..2cb031c6 100644 --- a/puppet/modules/site_couchdb/files/designs/messages/Message.json +++ b/puppet/modules/site_couchdb/files/designs/messages/Message.json @@ -2,14 +2,14 @@ "_id": "_design/Message", "language": "javascript", "views": { - "by_user_ids_to_show": { - "map": "function (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit(userId, 1);\n });\n }\n}\n", - "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" - }, "by_user_ids_to_show_and_created_at": { "map": "// not using at moment\n// call with something like Message.by_user_ids_to_show_and_created_at.startkey([user_id, start_date]).endkey([user_id,end_date])\nfunction (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit([userId, doc.created_at], 1);\n });\n }\n}\n", "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, + "by_user_ids_to_show": { + "map": "function (doc) {\n if (doc.type === 'Message' && doc.user_ids_to_show && Array.isArray(doc.user_ids_to_show)) {\n doc.user_ids_to_show.forEach(function (userId) {\n emit(userId, 1);\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, "all": { "map": " function(doc) {\n if (doc['type'] == 'Message') {\n emit(doc._id, null);\n }\n }\n" } diff --git a/puppet/modules/site_couchdb/files/designs/shared/docs.json b/puppet/modules/site_couchdb/files/designs/shared/docs.json deleted file mode 100644 index 004180cd..00000000 --- a/puppet/modules/site_couchdb/files/designs/shared/docs.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "_id": "_design/docs", - "views": { - "get": { - "map": "function(doc) {\n if (doc.u1db_rev) {\n var is_tombstone = true;\n var has_conflicts = false;\n if (doc._attachments) {\n if (doc._attachments.u1db_content)\n is_tombstone = false;\n if (doc._attachments.u1db_conflicts)\n has_conflicts = true;\n }\n emit(doc._id,\n {\n \"couch_rev\": doc._rev,\n \"u1db_rev\": doc.u1db_rev,\n \"is_tombstone\": is_tombstone,\n \"has_conflicts\": has_conflicts,\n }\n );\n }\n}\n" - } - } -}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/shared/syncs.json b/puppet/modules/site_couchdb/files/designs/shared/syncs.json deleted file mode 100644 index bab5622f..00000000 --- a/puppet/modules/site_couchdb/files/designs/shared/syncs.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "_id": "_design/syncs", - "updates": { - "put": "function(doc, req){\n if (!doc) {\n doc = {}\n doc['_id'] = 'u1db_sync_log';\n doc['syncs'] = [];\n }\n body = JSON.parse(req.body);\n // remove outdated info\n doc['syncs'] = doc['syncs'].filter(\n function (entry) {\n return entry[0] != body['other_replica_uid'];\n }\n );\n // store u1db rev\n doc['syncs'].push([\n body['other_replica_uid'],\n body['other_generation'],\n body['other_transaction_id']\n ]);\n return [doc, 'ok'];\n}\n\n" - }, - "views": { - "log": { - "map": "function(doc) {\n if (doc._id == 'u1db_sync_log') {\n if (doc.syncs)\n doc.syncs.forEach(function (entry) {\n emit(entry[0],\n {\n 'known_generation': entry[1],\n 'known_transaction_id': entry[2]\n });\n });\n }\n}\n" - } - } -}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/shared/transactions.json b/puppet/modules/site_couchdb/files/designs/shared/transactions.json deleted file mode 100644 index 106ad46c..00000000 --- a/puppet/modules/site_couchdb/files/designs/shared/transactions.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "_id": "_design/transactions", - "lists": { - "generation": "function(head, req) {\n var row;\n var rows=[];\n // fetch all rows\n while(row = getRow()) {\n rows.push(row);\n }\n if (rows.length > 0)\n send(JSON.stringify({\n \"generation\": rows.length,\n \"doc_id\": rows[rows.length-1]['id'],\n \"transaction_id\": rows[rows.length-1]['value']\n }));\n else\n send(JSON.stringify({\n \"generation\": 0,\n \"doc_id\": \"\",\n \"transaction_id\": \"\",\n }));\n}\n", - "trans_id_for_gen": "function(head, req) {\n var row;\n var rows=[];\n var i = 1;\n var gen = 1;\n if (req.query.gen)\n gen = parseInt(req.query['gen']);\n // fetch all rows\n while(row = getRow())\n rows.push(row);\n if (gen <= rows.length)\n send(JSON.stringify({\n \"generation\": gen,\n \"doc_id\": rows[gen-1]['id'],\n \"transaction_id\": rows[gen-1]['value'],\n }));\n else\n send('{}');\n}\n", - "whats_changed": "function(head, req) {\n var row;\n var gen = 1;\n var old_gen = 0;\n if (req.query.old_gen)\n old_gen = parseInt(req.query['old_gen']);\n send('{\"transactions\":[\\n');\n // fetch all rows\n while(row = getRow()) {\n if (gen > old_gen) {\n if (gen > old_gen+1)\n send(',\\n');\n send(JSON.stringify({\n \"generation\": gen,\n \"doc_id\": row[\"id\"],\n \"transaction_id\": row[\"value\"]\n }));\n }\n gen++;\n }\n send('\\n]}');\n}\n" - }, - "views": { - "log": { - "map": "function(doc) {\n if (doc.u1db_transactions)\n doc.u1db_transactions.forEach(function(t) {\n emit(t[0], // use timestamp as key so the results are ordered\n t[1]); // value is the transaction_id\n });\n}\n" - } - } -}
\ No newline at end of file diff --git a/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json b/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json index 578f632b..7ec24634 100644 --- a/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json +++ b/puppet/modules/site_couchdb/files/designs/tickets/Ticket.json @@ -22,8 +22,12 @@ "map": " function(doc) {\n if ((doc['type'] == 'Ticket') && (doc['is_open'] != null) && (doc['updated_at'] != null)) {\n emit([doc['is_open'], doc['updated_at']], 1);\n }\n }\n", "reduce": "_sum" }, - "by_includes_post_by_and_is_open_and_created_at": { - "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.is_open, doc.created_at], 1);\n }\n });\n }\n}\n", + "by_includes_post_by_and_updated_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.updated_at], 1);\n }\n });\n }\n}\n", + "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" + }, + "by_includes_post_by_and_created_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.created_at], 1);\n }\n });\n }\n}\n", "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, "by_includes_post_by": { @@ -34,12 +38,8 @@ "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.is_open, doc.updated_at], 1);\n }\n });\n }\n}\n", "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, - "by_includes_post_by_and_created_at": { - "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.created_at], 1);\n }\n });\n }\n}\n", - "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" - }, - "by_includes_post_by_and_updated_at": { - "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.updated_at], 1);\n }\n });\n }\n}\n", + "by_includes_post_by_and_is_open_and_created_at": { + "map": "function(doc) {\n var arr = {}\n if (doc['type'] == 'Ticket' && doc.comments) {\n doc.comments.forEach(function(comment){\n if (comment.posted_by && !arr[comment.posted_by]) {\n //don't add duplicates\n arr[comment.posted_by] = true;\n emit([comment.posted_by, doc.is_open, doc.created_at], 1);\n }\n });\n }\n}\n", "reduce": " function(key, values, rereduce) {\n return sum(values);\n }\n" }, "all": { diff --git a/puppet/modules/site_couchdb/manifests/designs.pp b/puppet/modules/site_couchdb/manifests/designs.pp index e5fd94c6..33687999 100644 --- a/puppet/modules/site_couchdb/manifests/designs.pp +++ b/puppet/modules/site_couchdb/manifests/designs.pp @@ -11,6 +11,14 @@ class site_couchdb::designs { mode => '0755' } + #cleanup leftovers from before soledad created its db + file { + '/srv/leap/couchdb/designs/shared/': + ensure => absent, + recurse => true, + force => true, + } + site_couchdb::upload_design { 'customers': design => 'customers/Customer.json'; 'identities': design => 'identities/Identity.json'; @@ -19,15 +27,6 @@ class site_couchdb::designs { 'users': design => 'users/User.json'; 'tmp_users': design => 'users/User.json'; 'invite_codes': design => 'invite_codes/InviteCode.json'; - 'shared_docs': - db => 'shared', - design => 'shared/docs.json'; - 'shared_syncs': - db => 'shared', - design => 'shared/syncs.json'; - 'shared_transactions': - db => 'shared', - design => 'shared/transactions.json'; } $sessions_db = rotated_db_name('sessions', 'monthly') diff --git a/puppet/modules/site_haproxy/files/haproxy-stats.cfg b/puppet/modules/site_haproxy/files/haproxy-stats.cfg deleted file mode 100644 index e6335ba2..00000000 --- a/puppet/modules/site_haproxy/files/haproxy-stats.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# provide access to stats for the nagios plugin -listen stats 127.0.0.1:8000 - mode http - stats enable - stats uri /haproxy - diff --git a/puppet/modules/site_haproxy/manifests/init.pp b/puppet/modules/site_haproxy/manifests/init.pp deleted file mode 100644 index b28ce80e..00000000 --- a/puppet/modules/site_haproxy/manifests/init.pp +++ /dev/null @@ -1,41 +0,0 @@ -class site_haproxy { - $haproxy = hiera('haproxy') - - class { 'haproxy': - enable => true, - manage_service => true, - global_options => { - 'log' => '127.0.0.1 local0', - 'maxconn' => '4096', - 'stats' => 'socket /var/run/haproxy.sock user haproxy group haproxy', - 'chroot' => '/usr/share/haproxy', - 'user' => 'haproxy', - 'group' => 'haproxy', - 'daemon' => '' - }, - defaults_options => { - 'log' => 'global', - 'retries' => '3', - 'option' => 'redispatch', - 'timeout connect' => '4000', - 'timeout client' => '20000', - 'timeout server' => '20000' - } - } - - # monitor haproxy - concat::fragment { 'stats': - target => '/etc/haproxy/haproxy.cfg', - order => '90', - source => 'puppet:///modules/site_haproxy/haproxy-stats.cfg'; - } - - # Template uses $haproxy - concat::fragment { 'leap_haproxy_webapp_couchdb': - target => '/etc/haproxy/haproxy.cfg', - order => '20', - content => template('site_haproxy/haproxy.cfg.erb'), - } - - include site_check_mk::agent::haproxy -} diff --git a/puppet/modules/site_haproxy/templates/couch.erb b/puppet/modules/site_haproxy/templates/couch.erb deleted file mode 100644 index f42e8368..00000000 --- a/puppet/modules/site_haproxy/templates/couch.erb +++ /dev/null @@ -1,32 +0,0 @@ -frontend couch - bind localhost:<%= @listen_port %> - mode http - option httplog - option dontlognull - option http-server-close # use client keep-alive, but close server connection. - use_backend couch_read if METH_GET - default_backend couch_write - -backend couch_write - mode http - balance roundrobin - option httpchk GET / # health check using simple get to root - option allbackups # balance among all backups, not just one. - default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 -<%- @servers.sort.each do |name,server| -%> -<%- next unless server['writable'] -%> - # <%=name%> - server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check -<%- end -%> - -backend couch_read - mode http - balance roundrobin - option httpchk GET / # health check using simple get to root - option allbackups # balance among all backups, not just one. - default-server inter 3000 fastinter 1000 downinter 1000 rise 2 fall 1 -<%- @servers.sort.each do |name,server| -%> - # <%=name%> - server couchdb_<%=server['port']%> <%=server['host']%>:<%=server['port']%> <%='backup' if server['backup']%> weight <%=server['weight']%> check -<%- end -%> - diff --git a/puppet/modules/site_haproxy/templates/haproxy.cfg.erb b/puppet/modules/site_haproxy/templates/haproxy.cfg.erb deleted file mode 100644 index 8311b1a5..00000000 --- a/puppet/modules/site_haproxy/templates/haproxy.cfg.erb +++ /dev/null @@ -1,11 +0,0 @@ -<%- @haproxy.each do |frontend, options| -%> -<%- if options['servers'] -%> - -## -## <%= frontend %> -## - -<%= scope.function_templatewlv(["site_haproxy/#{frontend}.erb", options]) %> -<%- end -%> -<%- end -%> - diff --git a/puppet/modules/site_mx/manifests/init.pp b/puppet/modules/site_mx/manifests/init.pp index c910a45a..28a01d4a 100644 --- a/puppet/modules/site_mx/manifests/init.pp +++ b/puppet/modules/site_mx/manifests/init.pp @@ -13,11 +13,12 @@ class site_mx { include ::site_stunnel include ::site_postfix::mx - include ::site_haproxy include ::site_shorewall::mx include ::site_shorewall::service::smtp include ::leap_mx include ::site_check_mk::agent::mx # install twisted from jessie backports include ::site_apt::preferences::twisted + # install python-cryptography from jessie backports + include ::site_apt::preferences::python_cryptography } diff --git a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg index 62f26f2c..1a9f266e 100644 --- a/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg +++ b/puppet/modules/site_nagios/files/configs/Debian/nagios.cfg @@ -773,7 +773,7 @@ accept_passive_host_checks=1 # service notifications when it is initially (re)started. # Values: 1 = enable notifications, 0 = disable notifications -enable_notifications=1 +enable_notifications=0 @@ -1299,4 +1299,3 @@ host_perfdata_file_template=DATATYPE::HOSTPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$H host_perfdata_file_mode=a host_perfdata_file_processing_interval=15 host_perfdata_file_processing_command=process-host-perfdata-file-pnp4nagios-bulk-npcd - diff --git a/puppet/modules/site_nagios/manifests/server/apache.pp b/puppet/modules/site_nagios/manifests/server/apache.pp index 82962e89..98d38122 100644 --- a/puppet/modules/site_nagios/manifests/server/apache.pp +++ b/puppet/modules/site_nagios/manifests/server/apache.pp @@ -17,9 +17,6 @@ class site_nagios::server::apache { include apache::module::php5 include apache::module::cgi - # apache >= 2.4, debian jessie - if ( $::lsbdistcodename == 'jessie' ) { - include apache::module::authn_core - } + include apache::module::authn_core } diff --git a/puppet/modules/site_nickserver/manifests/init.pp b/puppet/modules/site_nickserver/manifests/init.pp index ad97f829..48f7b73d 100644 --- a/puppet/modules/site_nickserver/manifests/init.pp +++ b/puppet/modules/site_nickserver/manifests/init.pp @@ -1,9 +1,8 @@ # -# TODO: currently, this is dependent on some things that are set up in +# TODO: currently, this is dependent on one thing that is set up in # site_webapp # -# (1) HAProxy -> couchdb -# (2) Apache +# (1) Apache # # It would be good in the future to make nickserver installable independently of # site_webapp. @@ -29,10 +28,11 @@ class site_nickserver { # the port that nickserver is actually running on $nickserver_local_port = '64250' - # couchdb is available on localhost via haproxy, which is bound to 4096. + # couchdb is available on localhost: + # - When couchdb is running on a different node: Via stunnel, which is bound to 4000. + # - When couchdb is running on the same node: On port 5984 $couchdb_host = 'localhost' - # See site_webapp/templates/haproxy_couchdb.cfg.erg - $couchdb_port = '4096' + $couchdb_port = $nickserver['couchdb_port'] $sources = hiera('sources') @@ -61,21 +61,30 @@ class site_nickserver { require => Group['nickserver']; } + # Eariler we used bundle install without --deployment + exec { 'clean_git_repo': + cwd => '/srv/leap/nickserver', + user => 'nickserver', + command => '/usr/bin/git checkout Gemfile.lock', + onlyif => '/usr/bin/git status | /bin/grep -q "modified: *Gemfile.lock"', + require => Package['git'] + } + vcsrepo { '/srv/leap/nickserver': - ensure => present, + ensure => latest, revision => $sources['nickserver']['revision'], provider => $sources['nickserver']['type'], source => $sources['nickserver']['source'], owner => 'nickserver', group => 'nickserver', - require => [ User['nickserver'], Group['nickserver'] ], + require => [ User['nickserver'], Group['nickserver'], Exec['clean_git_repo'] ], notify => Exec['nickserver_bundler_update']; } exec { 'nickserver_bundler_update': cwd => '/srv/leap/nickserver', - command => '/bin/bash -c "/usr/bin/bundle check || /usr/bin/bundle install --path vendor/bundle"', - unless => '/usr/bin/bundle check', + command => '/usr/bin/bundle install --deployment', + unless => '/bin/bash -c "/usr/bin/bundle config --local frozen 1; /usr/bin/bundle check"', user => 'nickserver', timeout => 600, require => [ @@ -101,42 +110,26 @@ class site_nickserver { # NICKSERVER DAEMON # - file { - '/usr/bin/nickserver': - ensure => link, - target => '/srv/leap/nickserver/bin/nickserver', - require => Vcsrepo['/srv/leap/nickserver']; - - '/etc/init.d/nickserver': - owner => root, - group => 0, - mode => '0755', - source => '/srv/leap/nickserver/dist/debian-init-script', - require => Vcsrepo['/srv/leap/nickserver']; + file { '/usr/bin/nickserver': + ensure => link, + target => '/srv/leap/nickserver/bin/nickserver', + require => Vcsrepo['/srv/leap/nickserver']; } - # register initscript at systemd on nodes newer than wheezy - # see https://leap.se/code/issues/7614 - case $::operatingsystemrelease { - /^7.*/: { } - default: { - exec { 'register_systemd_nickserver': - refreshonly => true, - command => '/bin/systemctl enable nickserver', - subscribe => File['/etc/init.d/nickserver'], - before => Service['nickserver']; - } - } + ::systemd::unit_file {'nickserver.service': + ensure => present, + source => '/srv/leap/nickserver/dist/nickserver.service', + subscribe => Vcsrepo['/srv/leap/nickserver'], + require => File['/usr/bin/nickserver']; } service { 'nickserver': - ensure => running, - enable => true, - hasrestart => true, - hasstatus => true, - require => [ - File['/etc/init.d/nickserver'], - File['/usr/bin/nickserver'], + ensure => running, + provider => 'systemd', + enable => true, + require => [ + Systemd::Unit_file['nickserver.service'], + Exec['systemctl-daemon-reload'], Class['Site_config::X509::Key'], Class['Site_config::X509::Cert'], Class['Site_config::X509::Ca'] ]; diff --git a/puppet/modules/site_openvpn/manifests/init.pp b/puppet/modules/site_openvpn/manifests/init.pp index f1ecefb9..ee7d6840 100644 --- a/puppet/modules/site_openvpn/manifests/init.pp +++ b/puppet/modules/site_openvpn/manifests/init.pp @@ -68,7 +68,7 @@ class site_openvpn { # find out the netmask in cidr format of the primary IF # thx to https://blog.kumina.nl/tag/puppet-tips-and-tricks/ # we can do this using an inline_template: - $factname_primary_netmask = "netmask_cidr_${::site_config::params::interface}" + $factname_primary_netmask = "netmask_${::site_config::params::interface}" $primary_netmask = inline_template('<%= scope.lookupvar(@factname_primary_netmask) %>') # deploy dh keys diff --git a/puppet/modules/site_openvpn/manifests/server_config.pp b/puppet/modules/site_openvpn/manifests/server_config.pp index 15e6fb38..f33ab17c 100644 --- a/puppet/modules/site_openvpn/manifests/server_config.pp +++ b/puppet/modules/site_openvpn/manifests/server_config.pp @@ -209,20 +209,15 @@ define site_openvpn::server_config( server => $openvpn_configname; } - # register openvpn services at systemd on nodes newer than wheezy + # register openvpn services at systemd # see https://leap.se/code/issues/7798 - case $::operatingsystemrelease { - /^7.*/: { } - default: { - exec { "enable_systemd_${openvpn_configname}": - refreshonly => true, - command => "/bin/systemctl enable openvpn@${openvpn_configname}", - subscribe => File["/etc/openvpn/${openvpn_configname}.conf"], - notify => Service["openvpn@${openvpn_configname}"]; - } - service { "openvpn@${openvpn_configname}": - ensure => running - } - } + exec { "enable_systemd_${openvpn_configname}": + refreshonly => true, + command => "/bin/systemctl enable openvpn@${openvpn_configname}", + subscribe => File["/etc/openvpn/${openvpn_configname}.conf"], + notify => Service["openvpn@${openvpn_configname}"]; + } + service { "openvpn@${openvpn_configname}": + ensure => running } } diff --git a/puppet/modules/site_openvpn/templates/add_gateway_ips.sh.erb b/puppet/modules/site_openvpn/templates/add_gateway_ips.sh.erb index e76b756b..f2d2bc70 100644 --- a/puppet/modules/site_openvpn/templates/add_gateway_ips.sh.erb +++ b/puppet/modules/site_openvpn/templates/add_gateway_ips.sh.erb @@ -1,11 +1,21 @@ #!/bin/sh -ip addr show dev <%= scope.lookupvar('site_config::params::interface') %> | grep -q <%= @openvpn_gateway_address %>/<%= @primary_netmask %> || +ip addr show dev <%= scope.lookupvar('site_config::params::interface') %> | grep -q "inet <%= @openvpn_gateway_address %>/" || ip addr add <%= @openvpn_gateway_address %>/<%= @primary_netmask %> dev <%= scope.lookupvar('site_config::params::interface') %> +EXITCODE=$? +if [ $EXITCODE != 0 ]; then + exit $EXITCODE +fi + <% if @openvpn_second_gateway_address %> -ip addr show dev <%= scope.lookupvar('site_config::params::interface') %> | grep -q <%= @openvpn_second_gateway_address %>/<%= @primary_netmask %> || +ip addr show dev <%= scope.lookupvar('site_config::params::interface') %> | grep -q "<%= @openvpn_second_gateway_address %>/" || ip addr add <%= @openvpn_second_gateway_address %>/<%= @primary_netmask %> dev <%= scope.lookupvar('site_config::params::interface') %> + +EXITCODE=$? +if [ $EXITCODE != 0 ]; then + exit $EXITCODE +fi <% end %> /bin/echo 1 > /proc/sys/net/ipv4/ip_forward diff --git a/puppet/modules/site_postfix/manifests/mx.pp b/puppet/modules/site_postfix/manifests/mx.pp index 2dac85f5..e94320c9 100644 --- a/puppet/modules/site_postfix/manifests/mx.pp +++ b/puppet/modules/site_postfix/manifests/mx.pp @@ -140,21 +140,13 @@ class site_postfix::mx { # greater verbosity for debugging, take out for production #include site_postfix::debug - case $::operatingsystemrelease { - /^7.*/: { - $smtpd_relay_restrictions='' - } - default: { - $smtpd_relay_restrictions=" -o smtpd_relay_restrictions=\$smtps_relay_restrictions\n" - } - } - $mastercf_tail = " smtps inet n - - - - smtpd -o smtpd_tls_wrappermode=yes -o smtpd_tls_security_level=encrypt -o tls_preempt_cipherlist=yes -${smtpd_relay_restrictions} -o smtpd_recipient_restrictions=\$smtps_recipient_restrictions + -o smtpd_relay_restrictions=\$smtps_relay_restrictions + -o smtpd_recipient_restrictions=\$smtps_recipient_restrictions -o smtpd_helo_restrictions=\$smtps_helo_restrictions -o smtpd_client_restrictions= -o cleanup_service_name=clean_smtps diff --git a/puppet/modules/site_sshd/manifests/init.pp b/puppet/modules/site_sshd/manifests/init.pp index a9202da4..7d5c728a 100644 --- a/puppet/modules/site_sshd/manifests/init.pp +++ b/puppet/modules/site_sshd/manifests/init.pp @@ -57,13 +57,8 @@ class site_sshd { # therefore we don't use it here, but include all other options # that would be applied by the 'hardened' parameter # not all options are available on wheezy - if ( $::lsbdistcodename == 'wheezy' ) { - $tail_additional_options = 'Ciphers aes256-ctr + $tail_additional_options = 'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes256-ctr MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' - } else { - $tail_additional_options = 'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes256-ctr -MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' - } ## ## SSHD SERVER CONFIGURATION diff --git a/puppet/modules/site_static/manifests/domain.pp b/puppet/modules/site_static/manifests/domain.pp index 6cf2c653..e456c94e 100644 --- a/puppet/modules/site_static/manifests/domain.pp +++ b/puppet/modules/site_static/manifests/domain.pp @@ -1,25 +1,30 @@ # configure static service for domain define site_static::domain ( - $ca_cert, + $ca_cert=undef, $key, $cert, $tls_only=true, $use_hidden_service=false, $locations=undef, $aliases=undef, - $apache_config=undef) { + $apache_config=undef, + $www_alias=false) { $domain = $name $base_dir = '/srv/static' - $cafile = "${cert}\n${ca_cert}" + if ($ca_cert) { + $certfile = "${cert}\n${ca_cert}" + } else { + $certfile = $cert + } if is_hash($locations) { create_resources(site_static::location, $locations) } x509::cert { $domain: - content => $cafile, + content => $certfile, notify => Service[apache] } x509::key { $domain: diff --git a/puppet/modules/site_static/manifests/hidden_service.pp b/puppet/modules/site_static/manifests/hidden_service.pp index f1f15f8e..c5d12c34 100644 --- a/puppet/modules/site_static/manifests/hidden_service.pp +++ b/puppet/modules/site_static/manifests/hidden_service.pp @@ -1,26 +1,32 @@ # create hidden service for static sites -class site_static::hidden_service { +class site_static::hidden_service ( $single_hop = false, $v3 = false ) { + Class['site_tor::hidden_service'] -> Class['site_static::hidden_service'] + include site_tor::hidden_service + + tor::daemon::hidden_service { 'static': + ports => [ '80 127.0.0.1:80'], + single_hop => $single_hop, + v3 => $v3 + } - include tor::daemon - tor::daemon::hidden_service { 'static': ports => [ '80 127.0.0.1:80'] } file { - '/var/lib/tor/webapp/': - ensure => directory, - owner => 'debian-tor', - group => 'debian-tor', - mode => '2700'; + '/var/lib/tor/static/': + ensure => directory, + owner => 'debian-tor', + group => 'debian-tor', + mode => '2700'; '/var/lib/tor/static/private_key': - ensure => present, - source => "/srv/leap/files/nodes/${::hostname}/tor.key", - owner => 'debian-tor', - group => 'debian-tor', - mode => '0600', - notify => Service['tor']; + ensure => present, + source => "/srv/leap/files/nodes/${::hostname}/tor.key", + owner => 'debian-tor', + group => 'debian-tor', + mode => '0600', + notify => Service['tor']; '/var/lib/tor/static/hostname': ensure => present, - content => "${::site_static::tor_domain}\n", + content => "${::site_static::onion_domain}\n", owner => 'debian-tor', group => 'debian-tor', mode => '0600', diff --git a/puppet/modules/site_static/manifests/init.pp b/puppet/modules/site_static/manifests/init.pp index dd3f912d..fdc5782f 100644 --- a/puppet/modules/site_static/manifests/init.pp +++ b/puppet/modules/site_static/manifests/init.pp @@ -7,11 +7,17 @@ class site_static { include site_config::x509::key include site_config::x509::ca_bundle - $static = hiera('static') - $domains = $static['domains'] - $formats = $static['formats'] - $bootstrap = $static['bootstrap_files'] - $tor = hiera('tor', false) + $services = hiera('services', []) + $static = hiera('static') + $domains = $static['domains'] + $formats = $static['formats'] + $bootstrap = $static['bootstrap_files'] + $tor = hiera('tor', false) + if $tor and member($services, 'tor_hidden_service') { + $onion_active = true + } else { + $onion_active = false + } file { '/srv/static/': @@ -54,10 +60,8 @@ class site_static { include site_config::ruby::dev if (member($formats, 'rack')) { - include site_apt::preferences::passenger class { 'passenger': manage_munin => false, - require => Class['site_apt::preferences::passenger'] } } @@ -67,16 +71,18 @@ class site_static { } package { 'zlib1g-dev': - ensure => installed + ensure => installed } } - if $tor { + if $onion_active { $hidden_service = $tor['hidden_service'] - $tor_domain = "${hidden_service['address']}.onion" - if $hidden_service['active'] { - include site_static::hidden_service + $onion_domain = "${hidden_service['address']}.onion" + class { 'site_static::hidden_service': + single_hop => $hidden_service['single_hop'], + v3 => $hidden_service['v3'] } + # Currently, we only support a single hidden service address per server. # So if there is more than one domain configured, then we need to make sure # we don't enable the hidden service for every domain. diff --git a/puppet/modules/site_static/templates/apache.conf.erb b/puppet/modules/site_static/templates/apache.conf.erb index dd04ca43..716df437 100644 --- a/puppet/modules/site_static/templates/apache.conf.erb +++ b/puppet/modules/site_static/templates/apache.conf.erb @@ -74,13 +74,15 @@ Require all granted </Directory> -<%- if @tor && (@always_use_hidden_service || @use_hidden_service) -%> +<%- if @onion_active && (@always_use_hidden_service || @use_hidden_service) -%> ## -## Tor +## Hidden Service ## <VirtualHost 127.0.0.1:80> - ServerName <%= @tor_domain %> - ServerAlias www.<%= @tor_domain %> + ServerName <%= @onion_domain %> +<%- if @www_alias -%> + ServerAlias www.<%= @onion_domain %> +<%- end -%> <IfModule mod_headers.c> Header set X-Frame-Options "deny" @@ -102,7 +104,9 @@ ## <VirtualHost *:80> ServerName <%= @domain %> +<%- if @www_alias -%> ServerAlias www.<%= @domain %> +<%- end -%> <%- @aliases && @aliases.each do |domain_alias| -%> ServerAlias <%= domain_alias %> <%- end -%> @@ -122,7 +126,9 @@ ## <VirtualHost *:443> ServerName <%= @domain %> +<%- if @www_alias -%> ServerAlias www.<%= @domain %> +<%- end -%> <%- @aliases && @aliases.each do |domain_alias| -%> ServerAlias <%= domain_alias %> <%- end -%> diff --git a/puppet/modules/site_stunnel/manifests/init.pp b/puppet/modules/site_stunnel/manifests/init.pp index a874721f..5f53d576 100644 --- a/puppet/modules/site_stunnel/manifests/init.pp +++ b/puppet/modules/site_stunnel/manifests/init.pp @@ -5,6 +5,15 @@ class site_stunnel { + # Install stunnel4 from jessie-backports because the + # jessie version randonly closes the connection prematurely + # see https://0xacab.org/leap/platform/issues/8746 + apt::preferences_snippet { 'stunnel4': + package => 'stunnel4', + release => "${::lsbdistcodename}-backports", + priority => 999; + } + # include the generic stunnel module # increase the number of open files to allow for 800 connections class { 'stunnel': default_extra => 'ulimit -n 4096' } @@ -45,4 +54,3 @@ class site_stunnel { include site_stunnel::override_service } - diff --git a/puppet/modules/site_tor/manifests/disable_exit.pp b/puppet/modules/site_tor/manifests/disable_exit.pp index 078f80ae..85c24bfc 100644 --- a/puppet/modules/site_tor/manifests/disable_exit.pp +++ b/puppet/modules/site_tor/manifests/disable_exit.pp @@ -1,7 +1,13 @@ +# ensure that the tor relay is not configured as an exit node class site_tor::disable_exit { tor::daemon::exit_policy { 'no_exit_at_all': reject => [ '*:*' ]; } +# In a future version of Tor, ExitRelay 0 may become the default when no ExitPolicy is given. + tor::daemon::snippet { + 'disable_exit': + content => 'ExitRelay 0'; + } } diff --git a/puppet/modules/site_tor/manifests/hidden_service.pp b/puppet/modules/site_tor/manifests/hidden_service.pp new file mode 100644 index 00000000..87a7b696 --- /dev/null +++ b/puppet/modules/site_tor/manifests/hidden_service.pp @@ -0,0 +1,13 @@ +# This class simply makes sure a base tor is installed and configured +# It doesn't configure any specific hidden service functionality, +# instead that is configured in site_webapp::hidden_service and +# site_static::hidden_service. +# +# Those could be factored out to make them more generic. +class site_tor::hidden_service { + tag 'leap_service' + Class['site_config::default'] -> Class['site_tor::hidden_service'] + + include site_config::default + include site_tor +} diff --git a/puppet/modules/site_tor/manifests/init.pp b/puppet/modules/site_tor/manifests/init.pp index 2207a5a9..5e209ba8 100644 --- a/puppet/modules/site_tor/manifests/init.pp +++ b/puppet/modules/site_tor/manifests/init.pp @@ -1,45 +1,14 @@ +# generic configuration needed for tor class site_tor { - tag 'leap_service' - Class['site_config::default'] -> Class['site_tor'] - $tor = hiera('tor') - $bandwidth_rate = $tor['bandwidth_rate'] - $tor_type = $tor['type'] - $nickname = $tor['nickname'] - $contact_emails = join($tor['contacts'],', ') - $family = $tor['family'] + # Ensure the tor version is the latest from backports + # see https://0xacab.org/leap/platform/issues/8783 + apt::preferences_snippet { 'tor': + package => 'tor', + release => "${::lsbdistcodename}-backports", + priority => 999, + before => Class['tor::daemon'] } - $address = hiera('ip_address') - - $openvpn = hiera('openvpn', undef) - if $openvpn { - $openvpn_ports = $openvpn['ports'] - } - else { - $openvpn_ports = [] - } - - include site_config::default - include tor::daemon - tor::daemon::relay { $nickname: - port => 9001, - address => $address, - contact_info => obfuscate_email($contact_emails), - bandwidth_rate => $bandwidth_rate, - my_family => $family - } - - if ( $tor_type == 'exit'){ - # Only enable the daemon directory if the node isn't also a webapp node - # or running openvpn on port 80 - if ! member($::services, 'webapp') and ! member($openvpn_ports, '80') { - tor::daemon::directory { $::hostname: port => 80 } - } - } - else { - include site_tor::disable_exit - } - - include site_shorewall::tor + class { 'tor::daemon': ensure_version => latest } } diff --git a/puppet/modules/site_tor/manifests/relay.pp b/puppet/modules/site_tor/manifests/relay.pp new file mode 100644 index 00000000..fcb83bc1 --- /dev/null +++ b/puppet/modules/site_tor/manifests/relay.pp @@ -0,0 +1,45 @@ +class site_tor::relay { + tag 'leap_service' + Class['site_config::default'] -> Class['site_tor::relay'] + + $tor = hiera('tor') + $bandwidth_rate = $tor['bandwidth_rate'] + $tor_type = $tor['type'] + $nickname = $tor['nickname'] + $contact_emails = join($tor['contacts'],', ') + $family = $tor['family'] + + $address = hiera('ip_address') + + $openvpn = hiera('openvpn', undef) + if $openvpn { + $openvpn_ports = $openvpn['ports'] + } + else { + $openvpn_ports = [] + } + + include site_config::default + include site_tor + + tor::daemon::relay { $nickname: + port => 9001, + address => $address, + contact_info => obfuscate_email($contact_emails), + bandwidth_rate => $bandwidth_rate, + my_family => $family + } + + if ( $tor_type == 'exit'){ + # Only enable the daemon directory if the node isn't also a webapp node + # or running openvpn on port 80 + if ! member($::services, 'webapp') and ! member($openvpn_ports, '80') { + tor::daemon::directory { $::hostname: port => 80 } + } + } + else { + include site_tor::disable_exit + } + + include site_shorewall::tor +} diff --git a/puppet/modules/site_webapp/manifests/couchdb.pp b/puppet/modules/site_webapp/manifests/couchdb.pp index 71450370..e1947048 100644 --- a/puppet/modules/site_webapp/manifests/couchdb.pp +++ b/puppet/modules/site_webapp/manifests/couchdb.pp @@ -1,9 +1,10 @@ +# Configures webapp couchdb config class site_webapp::couchdb { $webapp = hiera('webapp') - # haproxy listener on port localhost:4096, see site_webapp::haproxy + # stunnel endpoint on port localhost:4000 $couchdb_host = 'localhost' - $couchdb_port = '4096' + $couchdb_port = $webapp['couchdb_port'] $couchdb_webapp_user = $webapp['couchdb_webapp_user']['username'] $couchdb_webapp_password = $webapp['couchdb_webapp_user']['password'] $couchdb_admin_user = $webapp['couchdb_admin_user']['username'] @@ -22,8 +23,8 @@ class site_webapp::couchdb { # couchdb.admin.yml is a symlink to prevent the vcsrepo resource # from changing its user permissions every time. '/srv/leap/webapp/config/couchdb.admin.yml': - ensure => 'link', - target => '/etc/leap/couchdb.admin.yml', + ensure => 'link', + target => '/etc/leap/couchdb.admin.yml', require => Vcsrepo['/srv/leap/webapp']; '/etc/leap/couchdb.admin.yml': diff --git a/puppet/modules/site_webapp/manifests/hidden_service.pp b/puppet/modules/site_webapp/manifests/hidden_service.pp index d2662b65..290f9665 100644 --- a/puppet/modules/site_webapp/manifests/hidden_service.pp +++ b/puppet/modules/site_webapp/manifests/hidden_service.pp @@ -1,8 +1,10 @@ # Configure tor hidden service for webapp class site_webapp::hidden_service { + Class['site_tor::hidden_service'] -> Class['site_webapp::hidden_service'] + include site_tor::hidden_service $tor = hiera('tor') $hidden_service = $tor['hidden_service'] - $tor_domain = "${hidden_service['address']}.onion" + $onion_domain = "${hidden_service['address']}.onion" include site_apache::common include apache::module::headers @@ -10,27 +12,30 @@ class site_webapp::hidden_service { include apache::module::expires include apache::module::removeip - include tor::daemon - tor::daemon::hidden_service { 'webapp': ports => [ '80 127.0.0.1:80'] } + tor::daemon::hidden_service { 'webapp': + ports => [ '80 127.0.0.1:80'], + single_hop => $hidden_service['single_hop'], + v3 => $hidden_service['v3'] + } file { '/var/lib/tor/webapp/': - ensure => directory, - owner => 'debian-tor', - group => 'debian-tor', - mode => '2700'; + ensure => directory, + owner => 'debian-tor', + group => 'debian-tor', + mode => '2700'; '/var/lib/tor/webapp/private_key': - ensure => present, - source => "/srv/leap/files/nodes/${::hostname}/tor.key", - owner => 'debian-tor', - group => 'debian-tor', - mode => '0600', - notify => Service['tor']; + ensure => present, + source => "/srv/leap/files/nodes/${::hostname}/tor.key", + owner => 'debian-tor', + group => 'debian-tor', + mode => '0600', + notify => Service['tor']; '/var/lib/tor/webapp/hostname': ensure => present, - content => "${tor_domain}\n", + content => "${onion_domain}\n", owner => 'debian-tor', group => 'debian-tor', mode => '0600', diff --git a/puppet/modules/site_webapp/manifests/init.pp b/puppet/modules/site_webapp/manifests/init.pp index 83cf99a9..605d71b3 100644 --- a/puppet/modules/site_webapp/manifests/init.pp +++ b/puppet/modules/site_webapp/manifests/init.pp @@ -1,6 +1,7 @@ # configure webapp service class site_webapp { tag 'leap_service' + $services = hiera('services', []) $definition_files = hiera('definition_files') $provider = $definition_files['provider'] $eip_service = $definition_files['eip_service'] @@ -10,6 +11,7 @@ class site_webapp { $provider_domain = $node_domain['full_suffix'] $webapp = hiera('webapp') $api_version = $webapp['api_version'] + $secret_key_base = $webapp['secret_key_base'] $secret_token = $webapp['secret_token'] $tor = hiera('tor', false) $sources = hiera('sources') @@ -19,7 +21,6 @@ class site_webapp { include ::site_config::ruby::dev include ::site_webapp::apache include ::site_webapp::couchdb - include ::site_haproxy include ::site_webapp::cron include ::site_config::default include ::site_config::x509::cert @@ -106,7 +107,9 @@ class site_webapp { '/srv/leap/webapp/public/ca.crt': ensure => link, require => Vcsrepo['/srv/leap/webapp'], + # lint:ignore:variable_is_lowercase target => "${x509::variables::local_CAs}/${site_config::params::ca_name}.crt"; + # lint:endignore "/srv/leap/webapp/public/${api_version}": ensure => directory, @@ -175,11 +178,9 @@ class site_webapp { notify => Service['apache']; } - if $tor { + if $tor and member($services, 'tor_hidden_service') { $hidden_service = $tor['hidden_service'] - if $hidden_service['active'] { - include ::site_webapp::hidden_service - } + include ::site_webapp::hidden_service } diff --git a/puppet/modules/site_webapp/templates/config.yml.erb b/puppet/modules/site_webapp/templates/config.yml.erb index dd55d3e9..1a802f4c 100644 --- a/puppet/modules/site_webapp/templates/config.yml.erb +++ b/puppet/modules/site_webapp/templates/config.yml.erb @@ -8,6 +8,7 @@ production = { "force_ssl" => @webapp['secure'], "client_ca_key" => "%s/%s.key" % [scope.lookupvar('x509::variables::keys'), scope.lookupvar('site_config::params::client_ca_name')], "client_ca_cert" => "%s/%s.crt" % [scope.lookupvar('x509::variables::local_CAs'), scope.lookupvar('site_config::params::client_ca_name')], + "secret_key_base" => @secret_key_base, "secret_token" => @secret_token, "client_cert_lifespan" => cert_options['life_span'], "client_cert_bit_size" => cert_options['bit_size'].to_i, diff --git a/puppet/modules/soledad/manifests/server.pp b/puppet/modules/soledad/manifests/server.pp index 81f51188..3b6a2314 100644 --- a/puppet/modules/soledad/manifests/server.pp +++ b/puppet/modules/soledad/manifests/server.pp @@ -54,7 +54,10 @@ class soledad::server { package { $sources['soledad']['package']: ensure => $sources['soledad']['revision'], - require => Class['site_apt::leap_repo']; + require => [ + Class['site_apt::leap_repo'], + Package['ssl-cert'] + ]; } file { '/etc/default/soledad': diff --git a/puppet/modules/systemd/.fixtures.yml b/puppet/modules/systemd/.fixtures.yml new file mode 100644 index 00000000..1d455a31 --- /dev/null +++ b/puppet/modules/systemd/.fixtures.yml @@ -0,0 +1,4 @@ +--- +fixtures: + symlinks: + systemd: "#{source_dir}"
\ No newline at end of file diff --git a/puppet/modules/systemd/.gitrepo b/puppet/modules/systemd/.gitrepo index 1548a815..ea68e478 100644 --- a/puppet/modules/systemd/.gitrepo +++ b/puppet/modules/systemd/.gitrepo @@ -6,6 +6,6 @@ [subrepo] remote = https://leap.se/git/puppet_systemd branch = master - commit = 6d47fd4999fe03eba6fb11c4490dcbb90d937900 - parent = 56a771a3008d10720dd05fd815aeafbacdd1e08e + commit = f3c4059603a6ac19f132b0dc47b95e49d9ddc4ba + parent = 77d11c7ddeaeb123bf871bd2bfce0e5ace0c158e cmdver = 0.3.0 diff --git a/puppet/modules/systemd/.puppet-lint.rc b/puppet/modules/systemd/.puppet-lint.rc index d8f5c59e..e09d52f4 100644 --- a/puppet/modules/systemd/.puppet-lint.rc +++ b/puppet/modules/systemd/.puppet-lint.rc @@ -1,5 +1,5 @@ --fail-on-warnings --relative ---no-80chars +--no-140chars --no-documentation --no-class_inherits_from_params_class-check diff --git a/puppet/modules/systemd/.travis.yml b/puppet/modules/systemd/.travis.yml index 467045c5..1d1bedfc 100644 --- a/puppet/modules/systemd/.travis.yml +++ b/puppet/modules/systemd/.travis.yml @@ -1,22 +1,22 @@ --- language: ruby sudo: false +addons: + apt: + packages: + - libaugeas-dev + sources: + - augeas cache: bundler bundler_args: --without system_tests -script: ["bundle exec rake validate", "bundle exec rake lint", "bundle exec rake spec SPEC_OPTS='--format documentation'", "bundle exec rake metadata"] +script: ["bundle exec rake validate", "bundle exec rake lint", "bundle exec rake spec SPEC_OPTS='--format documentation'"] matrix: fast_finish: true include: - - rvm: 1.8.7 - env: PUPPET_GEM_VERSION="~> 3.0" FACTER_GEM_VERSION="~> 1.7.0" - - rvm: 1.9.3 - env: PUPPET_GEM_VERSION="~> 3.0" - - rvm: 2.0.0 - env: PUPPET_GEM_VERSION="~> 3.0" - - rvm: 2.0.0 - env: PUPPET_GEM_VERSION="~> 3.0" FUTURE_PARSER="yes" - - rvm: 2.1.6 + - rvm: 2.1.9 env: PUPPET_GEM_VERSION="~> 4.0" + - rvm: 2.3.1 + env: PUPPET_GEM_VERSION="~> 4" notifications: email: false deploy: @@ -29,4 +29,4 @@ deploy: # all_branches is required to use tags all_branches: true # Only publish if our main Ruby target builds - rvm: 1.9.3 + rvm: 2.1.9 diff --git a/puppet/modules/systemd/CHANGELOG.md b/puppet/modules/systemd/CHANGELOG.md index 11e84399..79b9e646 100644 --- a/puppet/modules/systemd/CHANGELOG.md +++ b/puppet/modules/systemd/CHANGELOG.md @@ -1,5 +1,22 @@ # Change Log +## [0.4.0](https://forge.puppetlabs.com/camptocamp/systemd/0.4.0) (2016-08-18) +[Full Changelog](https://github.com/camptocamp/puppet-systemd/compare/0.3.0...0.4.0) + +- Deprecate Ruby 1.8 tests +- Only use awk instead of grep and awk [\#9](https://github.com/camptocamp/puppet-systemd/pull/9) ([igalic](https://github.com/igalic)) +- Add LICENSE (fix #11) +- Add target param for the unit file [\#10](https://github.com/camptocamp/puppet-systemd/pull/10) ([tampakrap](https://github.com/tampakrap)) + +## [0.3.0](https://forge.puppetlabs.com/camptocamp/systemd/0.3.0) (2016-05-16) +[Full Changelog](https://github.com/camptocamp/puppet-systemd/compare/0.2.2...0.3.0) + +**Implemented enhancements:** + +- Shortcut for creating unit files / tmpfiles [\#4](https://github.com/camptocamp/puppet-systemd/pull/4) ([felixb](https://github.com/felixb)) +- Add systemd facts [\#6](https://github.com/camptocamp/puppet-systemd/pull/6) ([roidelapluie](https://github.com/roidelapluie)) + + ## [0.2.2](https://forge.puppetlabs.com/camptocamp/systemd/0.2.2) (2015-08-25) [Full Changelog](https://github.com/camptocamp/puppet-systemd/compare/0.2.1...0.2.2) @@ -60,6 +77,3 @@ \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* - - -\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*
\ No newline at end of file diff --git a/puppet/modules/systemd/Gemfile b/puppet/modules/systemd/Gemfile index 0cb59337..377d0c16 100644 --- a/puppet/modules/systemd/Gemfile +++ b/puppet/modules/systemd/Gemfile @@ -2,7 +2,7 @@ source ENV['GEM_SOURCE'] || "https://rubygems.org" group :development, :unit_tests do gem 'rake', :require => false - gem 'rspec', '< 3.2', :require => false if RUBY_VERSION =~ /^1.8/ + gem 'rspec', :require => false gem 'rspec-puppet', :require => false gem 'puppetlabs_spec_helper', :require => false gem 'metadata-json-lint', :require => false @@ -10,26 +10,26 @@ group :development, :unit_tests do gem 'puppet-lint-unquoted_string-check', :require => false gem 'puppet-lint-empty_string-check', :require => false gem 'puppet-lint-spaceship_operator_without_tag-check', :require => false - gem 'puppet-lint-variable_contains_upcase', :require => false gem 'puppet-lint-absolute_classname-check', :require => false gem 'puppet-lint-undef_in_function-check', :require => false gem 'puppet-lint-leading_zero-check', :require => false gem 'puppet-lint-trailing_comma-check', :require => false gem 'puppet-lint-file_ensure-check', :require => false gem 'puppet-lint-version_comparison-check', :require => false - gem 'puppet-lint-fileserver-check', :require => false gem 'puppet-lint-file_source_rights-check', :require => false gem 'puppet-lint-alias-check', :require => false gem 'rspec-puppet-facts', :require => false - gem 'github_changelog_generator', :require => false, :git => 'https://github.com/raphink/github-changelog-generator.git', :branch => 'dev/all_patches' if RUBY_VERSION !~ /^1.8/ - gem 'puppet-blacksmith', :require => false if RUBY_VERSION !~ /^1.8/ + gem 'ruby-augeas', :require => false + gem 'puppet-blacksmith', :require => false if RUBY_VERSION !~ /^1\./ + gem 'json_pure', '< 2.0.2', :require => false end group :system_tests do - gem 'beaker', :require => false - gem 'beaker-rspec', :require => false - gem 'beaker_spec_helper', :require => false - gem 'serverspec', :require => false + gem 'beaker', :require => false + gem 'beaker-rspec', '> 5', :require => false + gem 'beaker_spec_helper', :require => false + gem 'serverspec', :require => false + gem 'specinfra', :require => false end if facterversion = ENV['FACTER_GEM_VERSION'] diff --git a/puppet/modules/systemd/HISTORY.md b/puppet/modules/systemd/HISTORY.md index c7bf2b4e..aee8ad5e 100644 --- a/puppet/modules/systemd/HISTORY.md +++ b/puppet/modules/systemd/HISTORY.md @@ -1,3 +1,14 @@ +# Change Log + +## [0.3.0](https://forge.puppetlabs.com/camptocamp/systemd/0.3.0) (2016-05-16) +[Full Changelog](https://github.com/camptocamp/puppet-systemd/compare/0.2.2...0.3.0) + +**Implemented enhancements:** + +- Shortcut for creating unit files / tmpfiles [\#4](https://github.com/camptocamp/puppet-systemd/pull/4) ([felixb](https://github.com/felixb)) +- Add systemd facts [\#6](https://github.com/camptocamp/puppet-systemd/pull/6) ([roidelapluie](https://github.com/roidelapluie)) + + ## [0.2.2](https://forge.puppetlabs.com/camptocamp/systemd/0.2.2) (2015-08-25) [Full Changelog](https://github.com/camptocamp/puppet-systemd/compare/0.2.1...0.2.2) @@ -5,6 +16,7 @@ - Add 'systemd-tmpfiles-create' [\#1](https://github.com/camptocamp/puppet-systemd/pull/1) ([roidelapluie](https://github.com/roidelapluie)) + ## [0.2.1](https://forge.puppetlabs.com/camptocamp/systemd/0.2.1) (2015-08-21) [Full Changelog](https://github.com/camptocamp/puppet-systemd/compare/0.2.0...0.2.1) @@ -57,6 +69,3 @@ \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* - - -\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* diff --git a/puppet/modules/systemd/LICENSE b/puppet/modules/systemd/LICENSE new file mode 100644 index 00000000..8d968b6c --- /dev/null +++ b/puppet/modules/systemd/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/puppet/modules/systemd/README.md b/puppet/modules/systemd/README.md index f70bcb0c..51bf5cde 100644 --- a/puppet/modules/systemd/README.md +++ b/puppet/modules/systemd/README.md @@ -5,11 +5,23 @@ ## Overview -This module declares exec resources that you can use when you change systemd units or configuration files. +This module declares exec resources to create global sync points for reloading systemd. -## Examples +## Usage and examples -### systemctl --daemon-reload +There are two ways to use this module. + +### unit files + +Let this module handle file creation and systemd reloading. + +```puppet +::systemd::unit_file { 'foo.service': + source => "puppet:///modules/${module_name}/foo.service", +} +``` + +Or handle file creation yourself and trigger systemd. ```puppet include ::systemd @@ -23,7 +35,17 @@ file { '/usr/lib/systemd/system/foo.service': Exec['systemctl-daemon-reload'] ``` -### systemd-tmpfiles --create +### tmpfiles + +Let this module handle file creation and systemd reloading + +```puppet +::systemd::tmpfile { 'foo.conf': + source => "puppet:///modules/${module_name}/foo.conf", +} +``` + +Or handle file creation yourself and trigger systemd. ```puppet include ::systemd @@ -36,3 +58,24 @@ file { '/etc/tmpfiles.d/foo.conf': } ~> Exec['systemd-tmpfiles-create'] ``` + +### service limits + +Manage soft and hard limits on various resources for executed processes. + +```puppet +::systemd::service_limits { 'foo.service': + limits => { + LimitNOFILE => 8192, + LimitNPROC => 16384 + } +} +``` + +Or provide the configuration file yourself. Systemd reloading and restarting of the service are handled by the module. + +```puppet +::systemd::service_limits { 'foo.service': + source => "puppet:///modules/${module_name}/foo.conf", +} +``` diff --git a/puppet/modules/systemd/Rakefile b/puppet/modules/systemd/Rakefile index adcac180..aa7b8a15 100644 --- a/puppet/modules/systemd/Rakefile +++ b/puppet/modules/systemd/Rakefile @@ -4,20 +4,14 @@ require 'puppet-lint/tasks/puppet-lint' Rake::Task[:lint].clear PuppetLint::RakeTask.new :lint do |config| config.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp", "vendor/**/*.pp"] - config.disable_checks = ['80chars'] + config.disable_checks = ['140chars'] config.fail_on_warnings = true end PuppetSyntax.exclude_paths = ["spec/fixtures/**/*.pp", "vendor/**/*"] # Publishing tasks -unless RUBY_VERSION =~ /^1\.8/ +unless RUBY_VERSION =~ /^1\./ require 'puppet_blacksmith' require 'puppet_blacksmith/rake_tasks' - require 'github_changelog_generator/task' - GitHubChangelogGenerator::RakeTask.new :changelog do |config| - m = Blacksmith::Modulefile.new - config.future_release = m.version - config.release_url = "https://forge.puppetlabs.com/#{m.author}/#{m.name}/%s" - end end diff --git a/puppet/modules/systemd/lib/facter/systemd.rb b/puppet/modules/systemd/lib/facter/systemd.rb new file mode 100644 index 00000000..4361f775 --- /dev/null +++ b/puppet/modules/systemd/lib/facter/systemd.rb @@ -0,0 +1,35 @@ +# Fact: systemd +# +# Purpose: +# Determine whether SystemD is the init system on the node +# +# Resolution: +# Check the name of the process 1 (ps -p 1) +# +# Caveats: +# + +# Fact: systemd-version +# +# Purpose: +# Determine the version of systemd installed +# +# Resolution: +# Check the output of systemctl --version +# +# Caveats: +# + +Facter.add(:systemd) do + confine :kernel => :linux + setcode do + Facter::Util::Resolution.exec('ps -p 1 -o comm=') == 'systemd' + end +end + +Facter.add(:systemd_version) do + confine :systemd => true + setcode do + Facter::Util::Resolution.exec("systemctl --version | awk '/systemd/{ print $2 }'") + end +end diff --git a/puppet/modules/systemd/manifests/init.pp b/puppet/modules/systemd/manifests/init.pp index 5e6ad792..e669f093 100644 --- a/puppet/modules/systemd/manifests/init.pp +++ b/puppet/modules/systemd/manifests/init.pp @@ -1,4 +1,8 @@ -class systemd { +# -- Class systemd +# This module allows triggering systemd commands once for all modules +class systemd ( + $service_limits = {} +){ Exec { refreshonly => true, @@ -15,4 +19,6 @@ class systemd { command => 'systemd-tmpfiles --create', } + create_resources('systemd::service_limits', $service_limits, {}) + } diff --git a/puppet/modules/systemd/manifests/service_limits.pp b/puppet/modules/systemd/manifests/service_limits.pp new file mode 100644 index 00000000..a9cdc25a --- /dev/null +++ b/puppet/modules/systemd/manifests/service_limits.pp @@ -0,0 +1,50 @@ +# -- Define: systemd::service_limits +# Creates a custom config file and reloads systemd +define systemd::service_limits( + $ensure = file, + $path = '/etc/systemd/system', + $limits = undef, + $source = undef, + $restart_service = true +) { + include ::systemd + + if $limits { + validate_hash($limits) + $content = template('systemd/limits.erb') + } + else { + $content = undef + } + + if $limits and $source { + fail('You may not supply both limits and source parameters to systemd::service_limits') + } elsif $limits == undef and $source == undef { + fail('You must supply either the limits or source parameter to systemd::service_limits') + } + + file { "${path}/${title}.d/": + ensure => 'directory', + owner => 'root', + group => 'root', + } + -> + file { "${path}/${title}.d/limits.conf": + ensure => $ensure, + content => $content, + source => $source, + owner => 'root', + group => 'root', + mode => '0444', + notify => Exec['systemctl-daemon-reload'], + } + + if $restart_service { + exec { "systemctl restart ${title}": + path => $::path, + refreshonly => true, + subscribe => File["${path}/${title}.d/limits.conf"], + require => Exec['systemctl-daemon-reload'], + } + } +} diff --git a/puppet/modules/systemd/manifests/tmpfile.pp b/puppet/modules/systemd/manifests/tmpfile.pp new file mode 100644 index 00000000..c4d1a05f --- /dev/null +++ b/puppet/modules/systemd/manifests/tmpfile.pp @@ -0,0 +1,20 @@ +# -- Define: systemd::tmpfile +# Creates a tmpfile and reloads systemd +define systemd::tmpfile( + $ensure = file, + $path = '/etc/tmpfiles.d', + $content = undef, + $source = undef, +) { + include ::systemd + + file { "${path}/${title}": + ensure => $ensure, + content => $content, + source => $source, + owner => 'root', + group => 'root', + mode => '0444', + notify => Exec['systemd-tmpfiles-create'], + } +}
\ No newline at end of file diff --git a/puppet/modules/systemd/manifests/unit_file.pp b/puppet/modules/systemd/manifests/unit_file.pp new file mode 100644 index 00000000..94bc845b --- /dev/null +++ b/puppet/modules/systemd/manifests/unit_file.pp @@ -0,0 +1,22 @@ +# -- Define: systemd::unit_file +# Creates a unit file and reloads systemd +define systemd::unit_file( + $ensure = file, + $path = '/etc/systemd/system', + $content = undef, + $source = undef, + $target = undef, +) { + include ::systemd + + file { "${path}/${title}": + ensure => $ensure, + content => $content, + source => $source, + target => $target, + owner => 'root', + group => 'root', + mode => '0444', + notify => Exec['systemctl-daemon-reload'], + } +} diff --git a/puppet/modules/systemd/metadata.json b/puppet/modules/systemd/metadata.json index abdd481e..08951efb 100644 --- a/puppet/modules/systemd/metadata.json +++ b/puppet/modules/systemd/metadata.json @@ -1,6 +1,6 @@ { "name": "camptocamp-systemd", - "version": "0.2.2", + "version": "0.4.0", "author": "camptocamp", "summary": "Puppet Systemd module", "license": "Apache-2.0", diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/centos-5.yml b/puppet/modules/systemd/spec/acceptance/nodesets/centos-5.yml new file mode 100644 index 00000000..a26f27fc --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/centos-5.yml @@ -0,0 +1,16 @@ +HOSTS: + centos-5-x64: + default_apply_opts: + order: random + strict_variables: + platform: el-5-x86_64 + hypervisor : docker + image: tianon/centos:5.10 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'yum install -y crontabs tar wget which' + - 'sed -i -e "/mingetty/d" /etc/inittab' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/centos-6.yml b/puppet/modules/systemd/spec/acceptance/nodesets/centos-6.yml new file mode 100644 index 00000000..71e23cd8 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/centos-6.yml @@ -0,0 +1,17 @@ +HOSTS: + centos-6-x64: + default_apply_opts: + order: random + strict_variables: + platform: el-6-x86_64 + hypervisor : docker + image: centos:6 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'rm -rf /var/run/network/*' + - 'yum install -y crontabs tar wget' + - 'rm /etc/init/tty.conf' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/centos-7.yml b/puppet/modules/systemd/spec/acceptance/nodesets/centos-7.yml new file mode 100644 index 00000000..a8fa4686 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/centos-7.yml @@ -0,0 +1,15 @@ +HOSTS: + centos-7-x64: + default_apply_opts: + order: random + strict_variables: + platform: el-7-x86_64 + hypervisor : docker + image: centos:7 + docker_preserve_image: true + docker_cmd: '["/usr/sbin/init"]' + docker_image_commands: + - 'yum install -y crontabs tar wget iproute' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/debian-6.yml b/puppet/modules/systemd/spec/acceptance/nodesets/debian-6.yml new file mode 100644 index 00000000..d7b02756 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/debian-6.yml @@ -0,0 +1,15 @@ +HOSTS: + debian-6-x64: + default_apply_opts: + order: random + strict_variables: + platform: debian-6-amd64 + hypervisor : docker + image: debian/eol:squeeze + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'apt-get install -y cron locales-all net-tools wget' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/debian-7.yml b/puppet/modules/systemd/spec/acceptance/nodesets/debian-7.yml new file mode 100644 index 00000000..9591ea77 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/debian-7.yml @@ -0,0 +1,15 @@ +HOSTS: + debian-7-x64: + default_apply_opts: + order: random + strict_variables: + platform: debian-7-amd64 + hypervisor : docker + image: debian:7 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'apt-get install -y cron locales-all net-tools wget' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/debian-8.yml b/puppet/modules/systemd/spec/acceptance/nodesets/debian-8.yml new file mode 100644 index 00000000..5fb24c61 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/debian-8.yml @@ -0,0 +1,16 @@ +HOSTS: + debian-8-x64: + default_apply_opts: + order: random + strict_variables: + platform: debian-8-amd64 + hypervisor : docker + image: debian:8 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'apt-get install -y cron locales-all net-tools wget' + - 'rm -f /usr/sbin/policy-rc.d' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-12.04.yml b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-12.04.yml new file mode 100644 index 00000000..594e1771 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-12.04.yml @@ -0,0 +1,16 @@ +HOSTS: + ubuntu-1204-x64: + default_apply_opts: + order: random + strict_variables: + platform: ubuntu-12.04-amd64 + hypervisor : docker + image: ubuntu:12.04 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'apt-get install -y net-tools wget' + - 'locale-gen en_US.UTF-8' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-14.04.yml b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-14.04.yml new file mode 100644 index 00000000..2b293c99 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-14.04.yml @@ -0,0 +1,18 @@ +HOSTS: + ubuntu-1404-x64: + default_apply_opts: + order: random + strict_variables: + platform: ubuntu-14.04-amd64 + hypervisor : docker + image: ubuntu:14.04 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'rm /usr/sbin/policy-rc.d' + - 'rm /sbin/initctl; dpkg-divert --rename --remove /sbin/initctl' + - 'apt-get install -y net-tools wget' + - 'locale-gen en_US.UTF-8' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-14.10.yml b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-14.10.yml new file mode 100644 index 00000000..7ce09b2a --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-14.10.yml @@ -0,0 +1,18 @@ +HOSTS: + ubuntu-1410-x64: + default_apply_opts: + order: random + strict_variables: + platform: ubuntu-14.10-amd64 + hypervisor : docker + image: ubuntu:14.10 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'rm /usr/sbin/policy-rc.d' + - 'rm /sbin/initctl; dpkg-divert --rename --remove /sbin/initctl' + - 'apt-get install -y net-tools wget' + - 'locale-gen en_US.UTF-8' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-15.04.yml b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-15.04.yml new file mode 100644 index 00000000..329f3319 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-15.04.yml @@ -0,0 +1,16 @@ +HOSTS: + ubuntu-1504-x64: + default_apply_opts: + order: random + strict_variables: + platform: ubuntu-15.04-amd64 + hypervisor : docker + image: ubuntu:15.04 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'apt-get install -y net-tools wget' + - 'locale-gen en_US.UTF-8' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-15.10.yml b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-15.10.yml new file mode 100644 index 00000000..487795a3 --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-15.10.yml @@ -0,0 +1,16 @@ +HOSTS: + ubuntu-1510-x64: + default_apply_opts: + order: random + strict_variables: + platform: ubuntu-15.10-amd64 + hypervisor : docker + image: ubuntu:15.10 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'apt-get install -y net-tools wget' + - 'locale-gen en_US.UTF-8' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-16.04.yml b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-16.04.yml new file mode 100644 index 00000000..6c32b96d --- /dev/null +++ b/puppet/modules/systemd/spec/acceptance/nodesets/ubuntu-16.04.yml @@ -0,0 +1,16 @@ +HOSTS: + ubuntu-1604-x64: + default_apply_opts: + order: random + strict_variables: + platform: ubuntu-16.04-amd64 + hypervisor : docker + image: ubuntu:16.04 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + - 'apt-get install -y net-tools wget' + - 'locale-gen en_US.UTF-8' +CONFIG: + type: aio + log_level: debug diff --git a/puppet/modules/systemd/spec/defines/tmpfile_spec.rb b/puppet/modules/systemd/spec/defines/tmpfile_spec.rb new file mode 100644 index 00000000..4eb22acd --- /dev/null +++ b/puppet/modules/systemd/spec/defines/tmpfile_spec.rb @@ -0,0 +1,48 @@ +require 'spec_helper' + +describe 'systemd::tmpfile' do + + let(:facts) { { + :path => '/usr/bin', + } } + + context 'default params' do + + let(:title) { 'fancy.conf' } + + it 'creates the tmpfile' do + should contain_file('/etc/tmpfiles.d/fancy.conf').with({ + 'ensure' => 'file', + 'owner' => 'root', + 'group' => 'root', + 'mode' => '0444', + }) + end + + it 'triggers systemd daemon-reload' do + should contain_class('systemd') + should contain_file('/etc/tmpfiles.d/fancy.conf').with_notify("Exec[systemd-tmpfiles-create]") + end + end + + context 'with params' do + let(:title) { 'fancy.conf' } + + let(:params) { { + :ensure => 'absent', + :path => '/etc/tmpfiles.d/foo', + :content => 'some-content', + :source => 'some-source', + } } + + it 'creates the unit file' do + should contain_file('/etc/tmpfiles.d/foo/fancy.conf').with({ + 'ensure' => 'absent', + 'content' => 'some-content', + 'source' => 'some-source', + }) + end + + end + +end diff --git a/puppet/modules/systemd/spec/defines/unit_file_spec.rb b/puppet/modules/systemd/spec/defines/unit_file_spec.rb new file mode 100644 index 00000000..88a0122c --- /dev/null +++ b/puppet/modules/systemd/spec/defines/unit_file_spec.rb @@ -0,0 +1,50 @@ +require 'spec_helper' + +describe 'systemd::unit_file' do + + let(:facts) { { + :path => '/usr/bin', + } } + + context 'default params' do + + let(:title) { 'fancy.service' } + + it 'creates the unit file' do + should contain_file('/etc/systemd/system/fancy.service').with({ + 'ensure' => 'file', + 'owner' => 'root', + 'group' => 'root', + 'mode' => '0444', + }) + end + + it 'triggers systemd daemon-reload' do + should contain_class('systemd') + should contain_file('/etc/systemd/system/fancy.service').with_notify("Exec[systemctl-daemon-reload]") + end + end + + context 'with params' do + let(:title) { 'fancy.service' } + + let(:params) { { + :ensure => 'absent', + :path => '/usr/lib/systemd/system', + :content => 'some-content', + :source => 'some-source', + :target => 'some-target', + } } + + it 'creates the unit file' do + should contain_file('/usr/lib/systemd/system/fancy.service').with({ + 'ensure' => 'absent', + 'content' => 'some-content', + 'source' => 'some-source', + 'target' => 'some-target', + }) + end + + end + +end diff --git a/puppet/modules/systemd/spec/unit/facter/systemd_spec.rb b/puppet/modules/systemd/spec/unit/facter/systemd_spec.rb new file mode 100644 index 00000000..a7b62410 --- /dev/null +++ b/puppet/modules/systemd/spec/unit/facter/systemd_spec.rb @@ -0,0 +1,41 @@ +require "spec_helper" + +describe Facter::Util::Fact do + before { + Facter.clear + } + + describe "systemd" do + context 'returns true when systemd present' do + before do + Facter.fact(:kernel).stubs(:value).returns(:linux) + end + let(:facts) { {:kernel => :linux} } + it do + Facter::Util::Resolution.expects(:exec).with('ps -p 1 -o comm=').returns('systemd') + expect(Facter.value(:systemd)).to eq(true) + end + end + context 'returns false when systemd not present' do + before do + Facter.fact(:kernel).stubs(:value).returns(:linux) + end + let(:facts) { {:kernel => :linux} } + it do + Facter::Util::Resolution.expects(:exec).with('ps -p 1 -o comm=').returns('init') + expect(Facter.value(:systemd)).to eq(false) + end + end + + context 'returns nil when kernel is not linux' do + before do + Facter.fact(:kernel).stubs(:value).returns(:windows) + end + let(:facts) { {:kernel => :windows} } + it do + Facter::Util::Resolution.expects(:exec).with('ps -p 1 -o comm=').never + expect(Facter.value(:systemd)).to be_nil + end + end + end +end diff --git a/puppet/modules/systemd/spec/unit/facter/systemd_version_spec.rb b/puppet/modules/systemd/spec/unit/facter/systemd_version_spec.rb new file mode 100644 index 00000000..5007dc69 --- /dev/null +++ b/puppet/modules/systemd/spec/unit/facter/systemd_version_spec.rb @@ -0,0 +1,31 @@ +require "spec_helper" + +describe Facter::Util::Fact do + before { + Facter.clear + } + + describe "systemd_version" do + context 'returns version when systemd fact present' do + before do + Facter.fact(:systemd).stubs(:value).returns(true) + end + let(:facts) { {:systemd => true} } + it do + Facter::Util::Resolution.expects(:exec).with("systemctl --version | awk '/systemd/{ print $2 }'").returns('229') + expect(Facter.value(:systemd_version)).to eq('229') + end + end + context 'returns nil when systemd fact not present' do + before do + Facter.fact(:systemd).stubs(:value).returns(false) + end + let(:facts) { {:systemd => false } } + it do + Facter::Util::Resolution.stubs(:exec) + Facter::Util::Resolution.expects(:exec).with("systemctl --version | awk '/systemd/{ print $2 }'").never + expect(Facter.value(:systemd_version)).to eq(nil) + end + end + end +end diff --git a/puppet/modules/systemd/templates/limits.erb b/puppet/modules/systemd/templates/limits.erb new file mode 100644 index 00000000..3caf5867 --- /dev/null +++ b/puppet/modules/systemd/templates/limits.erb @@ -0,0 +1,26 @@ +# This file is created by Puppet +[Service] +<% +[ + 'LimitCPU', + 'LimitFSIZE', + 'LimitDATA', + 'LimitSTACK', + 'LimitCORE', + 'LimitRSS', + 'LimitNOFILE', + 'LimitAS', + 'LimitNPROC', + 'LimitMEMLOCK', + 'LimitLOCKS', + 'LimitSIGPENDING', + 'LimitMSGQUEUE', + 'LimitNICE', + 'LimitRTPRIO', + 'LimitRTTIME' +].each do |d| +if @limits[d] -%> +<%= d %>=<%= @limits[d] %> +<% +end +end %> diff --git a/puppet/modules/tor/.gitrepo b/puppet/modules/tor/.gitrepo index dfc1b3d9..ea3c1495 100644 --- a/puppet/modules/tor/.gitrepo +++ b/puppet/modules/tor/.gitrepo @@ -6,6 +6,6 @@ [subrepo] remote = https://leap.se/git/puppet_tor branch = master - commit = 9981a70f7ba1f9e4fe33e4eb46654295287c1fc1 - parent = 26aac7ccf240b06d65616bdd00ae472d980aaea9 - cmdver = 0.3.0 + commit = 4380e2eabd94d8f0df7f63c642dd46ec4783ef07 + parent = be4182d7227d57b4da20d088b4750c756f759888 + cmdver = 0.3.1 diff --git a/puppet/modules/tor/README b/puppet/modules/tor/README index 7777438a..188accac 100644 --- a/puppet/modules/tor/README +++ b/puppet/modules/tor/README @@ -113,7 +113,7 @@ Installing torsocks To install torsocks, simply include the 'torsocks' class in your manifests: - class { 'torsocks': } + class { 'tor::torsocks': } You can specify the $ensure_version class parameter to get a specific version installed. diff --git a/puppet/modules/tor/manifests/daemon/base.pp b/puppet/modules/tor/manifests/daemon/base.pp index 63d7bc4d..c0c82ac6 100644 --- a/puppet/modules/tor/manifests/daemon/base.pp +++ b/puppet/modules/tor/manifests/daemon/base.pp @@ -2,7 +2,7 @@ class tor::daemon::base inherits tor::base { # packages, user, group Service['tor'] { - subscribe => File[$tor::daemon::config_file], + subscribe => Concat[$tor::daemon::config_file], } Package[ 'tor' ] { @@ -49,18 +49,15 @@ class tor::daemon::base inherits tor::base { # tor configuration file concat { $tor::daemon::config_file: - mode => '0600', - owner => 'debian-tor', - group => 'debian-tor', + mode => '0600', + owner => 'debian-tor', + group => 'debian-tor', } # config file headers concat::fragment { '00.header': ensure => present, content => template('tor/torrc.header.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 00, target => $tor::daemon::config_file, } @@ -68,9 +65,6 @@ class tor::daemon::base inherits tor::base { # global configurations concat::fragment { '01.global': content => template('tor/torrc.global.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 01, target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/bridge.pp b/puppet/modules/tor/manifests/daemon/bridge.pp index 063f5656..83d74e07 100644 --- a/puppet/modules/tor/manifests/daemon/bridge.pp +++ b/puppet/modules/tor/manifests/daemon/bridge.pp @@ -8,9 +8,6 @@ define tor::daemon::bridge( concat::fragment { "10.bridge.${name}": ensure => $ensure, content => template('tor/torrc.bridge.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 10, target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/control.pp b/puppet/modules/tor/manifests/daemon/control.pp index 01726562..ee425f33 100644 --- a/puppet/modules/tor/manifests/daemon/control.pp +++ b/puppet/modules/tor/manifests/daemon/control.pp @@ -7,20 +7,20 @@ define tor::daemon::control( $cookie_auth_file_group_readable = '', $ensure = present ) { - if $cookie_authentication == '0' and $hashed_control_password == '' and $ensure != 'absent' { - fail('You need to define the tor control password') - } + if $cookie_authentication == '0' + and $hashed_control_password == '' + and $ensure != 'absent' { + fail('You need to define the tor control password') + } - if $cookie_authentication == 0 and ($cookie_auth_file != '' or $cookie_auth_file_group_readable != '') { - notice('You set a tor cookie authentication option, but do not have cookie_authentication on') - } + if $cookie_authentication == 0 + and ($cookie_auth_file != '' or $cookie_auth_file_group_readable != '') { + notice('You set a tor cookie authentication option, but do not have cookie_authentication on') # lint:ignore:80chars + } concat::fragment { '04.control': ensure => $ensure, content => template('tor/torrc.control.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0600', order => 04, target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/directory.pp b/puppet/modules/tor/manifests/daemon/directory.pp index d877a861..e2e405da 100644 --- a/puppet/modules/tor/manifests/daemon/directory.pp +++ b/puppet/modules/tor/manifests/daemon/directory.pp @@ -8,9 +8,6 @@ define tor::daemon::directory ( concat::fragment { '06.directory': ensure => $ensure, content => template('tor/torrc.directory.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 06, target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/dns.pp b/puppet/modules/tor/manifests/daemon/dns.pp index 4677f24d..e8d4fc88 100644 --- a/puppet/modules/tor/manifests/daemon/dns.pp +++ b/puppet/modules/tor/manifests/daemon/dns.pp @@ -7,9 +7,6 @@ define tor::daemon::dns( concat::fragment { "08.dns.${name}": ensure => $ensure, content => template('tor/torrc.dns.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => '08', target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/exit_policy.pp b/puppet/modules/tor/manifests/daemon/exit_policy.pp index f459ece7..df0fb999 100644 --- a/puppet/modules/tor/manifests/daemon/exit_policy.pp +++ b/puppet/modules/tor/manifests/daemon/exit_policy.pp @@ -8,9 +8,6 @@ define tor::daemon::exit_policy( concat::fragment { "07.exit_policy.${name}": ensure => $ensure, content => template('tor/torrc.exit_policy.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 07, target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/hidden_service.pp b/puppet/modules/tor/manifests/daemon/hidden_service.pp index c8272116..d91bdc89 100644 --- a/puppet/modules/tor/manifests/daemon/hidden_service.pp +++ b/puppet/modules/tor/manifests/daemon/hidden_service.pp @@ -1,17 +1,22 @@ # hidden services definition define tor::daemon::hidden_service( - $ports = [], - $data_dir = $tor::daemon::data_dir, - $ensure = present ) { + $ports = [], + $single_hop = false, + $v3 = false, + $data_dir = $tor::daemon::data_dir, + $ensure = present ) { + + + if $single_hop { + file { "${$data_dir}/${$name}/onion_service_non_anonymous": + ensure => 'present', + } + } concat::fragment { "05.hidden_service.${name}": ensure => $ensure, content => template('tor/torrc.hidden_service.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 05, target => $tor::daemon::config_file, } } - diff --git a/puppet/modules/tor/manifests/daemon/map_address.pp b/puppet/modules/tor/manifests/daemon/map_address.pp index 270eac21..ac624a0a 100644 --- a/puppet/modules/tor/manifests/daemon/map_address.pp +++ b/puppet/modules/tor/manifests/daemon/map_address.pp @@ -7,9 +7,6 @@ define tor::daemon::map_address( concat::fragment { "08.map_address.${name}": ensure => $ensure, content => template('tor/torrc.map_address.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => '08', target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/relay.pp b/puppet/modules/tor/manifests/daemon/relay.pp index ff528937..555587cd 100644 --- a/puppet/modules/tor/manifests/daemon/relay.pp +++ b/puppet/modules/tor/manifests/daemon/relay.pp @@ -33,9 +33,6 @@ define tor::daemon::relay( concat::fragment { '03.relay': ensure => $ensure, content => template('tor/torrc.relay.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 03, target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/snippet.pp b/puppet/modules/tor/manifests/daemon/snippet.pp index b9089b40..7e1494c5 100644 --- a/puppet/modules/tor/manifests/daemon/snippet.pp +++ b/puppet/modules/tor/manifests/daemon/snippet.pp @@ -6,9 +6,6 @@ define tor::daemon::snippet( concat::fragment { "99.snippet.${name}": ensure => $ensure, content => $content, - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 99, target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/socks.pp b/puppet/modules/tor/manifests/daemon/socks.pp index 910461c9..54c8b6a2 100644 --- a/puppet/modules/tor/manifests/daemon/socks.pp +++ b/puppet/modules/tor/manifests/daemon/socks.pp @@ -6,9 +6,6 @@ define tor::daemon::socks( concat::fragment { '02.socks': content => template('tor/torrc.socks.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => 02, target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/daemon/transparent.pp b/puppet/modules/tor/manifests/daemon/transparent.pp index 65d744f4..6ac7b44c 100644 --- a/puppet/modules/tor/manifests/daemon/transparent.pp +++ b/puppet/modules/tor/manifests/daemon/transparent.pp @@ -7,9 +7,6 @@ define tor::daemon::transparent( concat::fragment { "09.transparent.${name}": ensure => $ensure, content => template('tor/torrc.transparent.erb'), - owner => 'debian-tor', - group => 'debian-tor', - mode => '0644', order => '09', target => $tor::daemon::config_file, } diff --git a/puppet/modules/tor/manifests/munin.pp b/puppet/modules/tor/manifests/munin.pp index 4412337a..2a01175c 100644 --- a/puppet/modules/tor/manifests/munin.pp +++ b/puppet/modules/tor/manifests/munin.pp @@ -8,7 +8,7 @@ class tor::munin { } Munin::Plugin::Deploy { - config => "user debian-tor\n env.cookiefile /var/run/tor/control.authcookie\n env.port 19051" + config => "user debian-tor\n env.cookiefile /var/run/tor/control.authcookie\n env.port 19051" # lint:ignore:80chars } munin::plugin::deploy { 'tor_connections': diff --git a/puppet/modules/tor/manifests/repo.pp b/puppet/modules/tor/manifests/repo.pp index f6255995..95492191 100644 --- a/puppet/modules/tor/manifests/repo.pp +++ b/puppet/modules/tor/manifests/repo.pp @@ -1,3 +1,4 @@ +# setup repository for tor class tor::repo ( $ensure = present, $source_name = 'torproject.org', @@ -10,7 +11,7 @@ class tor::repo ( class { 'tor::repo::debian': } } default: { - fail("Unsupported managed repository for osfamily: ${::osfamily}, operatingsystem: ${::operatingsystem}, module ${module_name} currently only supports managing repos for osfamily Debian and Ubuntu") + fail("Unsupported managed repository for osfamily: ${::osfamily}, operatingsystem: ${::operatingsystem}, module ${module_name} currently only supports managing repos for osfamily Debian and Ubuntu") # lint:ignore:80chars } } } diff --git a/puppet/modules/tor/manifests/repo/debian.pp b/puppet/modules/tor/manifests/repo/debian.pp index 174c3310..81976a2e 100644 --- a/puppet/modules/tor/manifests/repo/debian.pp +++ b/puppet/modules/tor/manifests/repo/debian.pp @@ -1,6 +1,6 @@ # PRIVATE CLASS: do not use directly class tor::repo::debian inherits tor::repo { - apt::source { $source_name: + apt::source { $tor::repo::source_name: ensure => $::tor::repo::ensure, location => $::tor::repo::location, key => $::tor::repo::key, diff --git a/puppet/modules/tor/templates/torrc.directory.erb b/puppet/modules/tor/templates/torrc.directory.erb index 1af9f40f..23ed3392 100644 --- a/puppet/modules/tor/templates/torrc.directory.erb +++ b/puppet/modules/tor/templates/torrc.directory.erb @@ -1,11 +1,11 @@ # directory listing -<% if port != '0' -%> +<% if @port != '0' -%> DirPort <%= @port %> <% end -%> -<% listen_addresses.each do |listen_address| -%> +<% @listen_addresses.each do |listen_address| -%> DirListenAddress <%= listen_address %> <% end -%> <% if @port_front_page != '' -%> -DirPortFrontPage <%= port_front_page %> +DirPortFrontPage <%= @port_front_page %> <%- end -%> diff --git a/puppet/modules/tor/templates/torrc.global.erb b/puppet/modules/tor/templates/torrc.global.erb index f577673d..a02afc8e 100644 --- a/puppet/modules/tor/templates/torrc.global.erb +++ b/puppet/modules/tor/templates/torrc.global.erb @@ -12,8 +12,8 @@ Log notice syslog Log <%= log_rule %> <% end -%> <% end -%> -<%- if @safe_logging != 1 then -%> -SafeLogging <%= @safe_logging %> +<%- if (v=scope.lookupvar('tor::daemon::safe_logging')) != '1' then -%> +SafeLogging <%= v %> <%- end -%> <% if (v=scope.lookupvar('tor::daemon::automap_hosts_on_resolve')) != '0' -%> diff --git a/puppet/modules/tor/templates/torrc.hidden_service.erb b/puppet/modules/tor/templates/torrc.hidden_service.erb index 4dec0b25..8a691c6b 100644 --- a/puppet/modules/tor/templates/torrc.hidden_service.erb +++ b/puppet/modules/tor/templates/torrc.hidden_service.erb @@ -1,6 +1,20 @@ +<% if @single_hop != false %> +HiddenServiceSingleHopMode 1 +HiddenServiceNonAnonymousMode 1 +SOCKSPort 0 +<% end %> + # hidden service <%= @name %> HiddenServiceDir <%= @data_dir %>/<%= @name %> <% @ports.each do |port| -%> HiddenServicePort <%= port %> <% end -%> +<% if @v3 != false %> +# hidden service v3 static +HiddenServiceDir <%= @data_dir %>/<%= @name -%>3 +HiddenServiceVersion 3 +<% @ports.each do |port| -%> +HiddenServicePort <%= port %> +<% end -%> +<% end -%> diff --git a/tests/example-provider/Vagrantfile b/tests/example-provider/Vagrantfile index 1e410f5e..e909e79b 100644 --- a/tests/example-provider/Vagrantfile +++ b/tests/example-provider/Vagrantfile @@ -42,6 +42,10 @@ Vagrant.configure("2") do |config| config.ssh.username = "vagrant" + # Enable private networking so the box can be accessed directly, + # not only via port forwaring + config.vm.network "private_network", type: "dhcp" + # forward leap_web ports config.vm.network "forwarded_port", guest: 443, host:4443 # forward pixelated ports diff --git a/tests/example-provider/vagrant/add-pixelated.sh b/tests/example-provider/vagrant/add-pixelated.sh index f9908947..38a7ea47 100755 --- a/tests/example-provider/vagrant/add-pixelated.sh +++ b/tests/example-provider/vagrant/add-pixelated.sh @@ -6,8 +6,8 @@ cd "$PROVIDERDIR" -if ! git submodule status files/puppet/modules/pixelated > /dev/null 2>&1; then - git submodule add https://github.com/pixelated/puppet-pixelated.git files/puppet/modules/pixelated +if ! [ -d files/puppet/modules/pixelated ]; then + git clone https://github.com/pixelated/puppet-pixelated.git files/puppet/modules/pixelated fi echo '{}' > services/pixelated.json @@ -26,7 +26,7 @@ $LEAP $OPTS -v 2 test --continue echo -e '\n===========================================================================================================\n\n' echo -e 'You are now ready to use your vagrant Pixelated provider.\n' -echo -e 'The LEAP webapp is available at https://localhost:4443. Use it to register an account before using the Pixelated Useragent.\n' -echo -e 'The Pixelated Useragent is available at https://localhost:8080\n' +echo -e 'The LEAP webapp is available at https://localhost:4443. Use it to register an account before using the Pixelated User Agent.\n' +echo -e 'The Pixelated User Agent is available at https://localhost:8080\n' echo -e 'Please add an exception for both sites in your browser dialog to allow the self-signed certificate.\n' diff --git a/tests/example-provider/vagrant/configure-leap.sh b/tests/example-provider/vagrant/configure-leap.sh index fd34d7ea..7a1efc71 100755 --- a/tests/example-provider/vagrant/configure-leap.sh +++ b/tests/example-provider/vagrant/configure-leap.sh @@ -1,45 +1,53 @@ -#!/bin/bash +#!/bin/sh +# Exit on failure +set -e +# shellcheck disable=SC1091 . /vagrant/vagrant/vagrant.config echo '===============================================' -echo 'configuring leap' +echo "Configuring LEAP in ${PROVIDERDIR}" echo '===============================================' # purge $PROVIDERDIR so this script can be run multiple times -[ -e $PROVIDERDIR ] && rm -rf $PROVIDERDIR +[ -e "$PROVIDERDIR" ] && rm -rf "$PROVIDERDIR" -mkdir -p $PROVIDERDIR -chown ${USER}:${USER} ${PROVIDERDIR} -cd $PROVIDERDIR +mkdir -p "$PROVIDERDIR" +chown "${USER}:${USER}" "${PROVIDERDIR}" +cd "$PROVIDERDIR" || exit -$LEAP $OPTS new --contacts "$contacts" --domain "$provider_domain" --name "$provider_name" --platform="$PLATFORMDIR" . -echo -e '\n@log = "./deploy.log"' >> Leapfile +$LEAP new --contacts "${contacts:?}" --domain "${provider_domain:?}" --name "${provider_name:?}" --platform="$PLATFORMDIR" . +printf '\n@log = "./deploy.log"' >> Leapfile -if [ ! -e /home/${USER}/.ssh/id_rsa ]; then - $SUDO ssh-keygen -f /home/${USER}/.ssh/id_rsa -P '' +if [ ! -e "/home/${USER}/.ssh/id_rsa" ]; then + $SUDO ssh-keygen -f "/home/${USER}/.ssh/id_rsa" -P '' [ -d /root/.ssh ] || mkdir /root/.ssh - cat /home/${USER}/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys + cat "/home/${USER}/.ssh/id_rsa.pub" >> /root/.ssh/authorized_keys fi -$SUDO mkdir -p ${PROVIDERDIR}/files/nodes/${NODE} +$SUDO mkdir -p "${PROVIDERDIR}/files/nodes/${NODE}" sh -c "cat /etc/ssh/ssh_host_rsa_key.pub | cut -d' ' -f1,2 >> $PROVIDERDIR/files/nodes/$NODE/${NODE}_ssh.pub" -chown ${USER}:${USER} ${PROVIDERDIR}/files/nodes/${NODE}/${NODE}_ssh.pub +chown "${USER}:${USER}" "${PROVIDERDIR}/files/nodes/${NODE}/${NODE}_ssh.pub" -$LEAP $OPTS add-user --self -$LEAP $OPTS cert ca -$LEAP $OPTS cert csr -$LEAP $OPTS node add $NODE ip_address:"$(facter ipaddress)" couch.mode:plain services:"$services" tags:production -echo '{ "webapp": { "admins": ["testadmin"] } }' > services/webapp.json +$LEAP add-user --self +$LEAP cert ca +$LEAP cert csr + +# Try to see if there's a private IP for eth1 +# Otherwise take eth0 +# (virtualbox and libvirt backends behave differenently setting up +# direct accessible private networks. +# see https://www.vagrantup.com/docs/networking/private_network.html +IP="$(facter ipaddress_eth1)" +[ "$IP" = '' ] && IP="$(facter ipaddress_eth0)" +$LEAP node add "$NODE" ip_address:"${IP}" couch.mode:plain services:"${services:?}" tags:production -$LEAP $OPTS compile +echo '{ "webapp": { "admins": ["testadmin"] } }' > services/webapp.json -$GIT init -$GIT add . -$GIT commit -m'configured provider' +$LEAP compile -$LEAP $OPTS node init $NODE +$LEAP node init "$NODE" if [ $? -eq 1 ]; then echo 'node init failed' exit 1 @@ -50,10 +58,7 @@ fi # workaround is to install rake as gem gem install rake -$LEAP $OPTS -v 2 deploy - -$GIT add . -$GIT commit -m'initialized and deployed provider' +$LEAP -v 2 deploy # Vagrant: leap_mx fails to start on jessie # https://leap.se/code/issues/7755 @@ -69,7 +74,7 @@ echo '===============================================' echo 'testing the platform' echo '===============================================' -$LEAP $OPTS -v 2 test --continue +$LEAP -v 2 test --continue echo '===============================================' echo 'setting node to demo-mode' @@ -80,13 +85,13 @@ postconf -e default_transport='error: in demo mode' curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testuser&user%5Bpassword_salt%5D=7d4880237a038e0e&user%5Bpassword_verifier%5D=b98dc393afcd16e5a40fb57ce9cddfa6a978b84be326196627c111d426cada898cdaf3a6427e98b27daf4b0ed61d278bc856515aeceb2312e50c8f816659fcaa4460d839a1e2d7ffb867d32ac869962061368141c7571a53443d58dc84ca1fca34776894414c1090a93e296db6cef12c2cc3f7a991b05d49728ed358fd868286" curl -s -k https://localhost/1/users.json -d "user%5Blogin%5D=testadmin&user%5Bpassword_salt%5D=ece1c457014d8282&user%5Bpassword_verifier%5D=9654d93ab409edf4ff1543d07e08f321107c3fd00de05c646c637866a94f28b3eb263ea9129dacebb7291b3374cc6f0bf88eb3d231eb3a76eed330a0e8fd2a5c477ed2693694efc1cc23ae83c2ae351a21139701983dd595b6c3225a1bebd2a4e6122f83df87606f1a41152d9890e5a11ac3749b3bfcf4407fc83ef60b4ced68" -echo -e '\n===========================================================================================================\n\n' -echo -e 'You are now ready to use your local LEAP provider.\n' +printf '\n===========================================================================================================\n\n' +printf 'You are now ready to use your local LEAP provider.\n' echo 'If you want to use the *Bitmask client* with your provider, please update your /etc/hosts with following dns overrides:' $LEAP list --print ip_address,domain.full,dns.aliases | sed 's/^.* //' | sed 's/, null//g' | tr -d '\]\[",' echo 'Please see https://leap.se/en/docs/platform/tutorials/vagrant#use-the-bitmask-client-to-do-an-initial-soledad-sync for more details how to use and test your LEAP provider.' -echo -e "\nIf you don't want to use the Bitmask client, please ignore the above instructions.\n" -echo -e 'The LEAP webapp is now available at https://localhost:4443\n' -echo -e 'Please add an exception in your browser dialog to allow the self-signed certificate.\n' +printf "\nIf you don't want to use the Bitmask client, please ignore the above instructions.\n" +printf 'The LEAP webapp is now available at https://localhost:4443\n' +printf 'Please add an exception in your browser dialog to allow the self-signed certificate.\n' diff --git a/tests/example-provider/vagrant/install-platform.pp b/tests/example-provider/vagrant/install-platform.pp index 223853c1..9cefcf7c 100755 --- a/tests/example-provider/vagrant/install-platform.pp +++ b/tests/example-provider/vagrant/install-platform.pp @@ -10,6 +10,3 @@ class { '::leap::cli::install': file { [ '/srv/leap', '/srv/leap/configuration', '/var/log/leap' ]: ensure => directory } - -# install prerequisites for configuring the provider -include ::git diff --git a/tests/example-provider/vagrant/vagrant.config b/tests/example-provider/vagrant/vagrant.config index 60d2a52c..07222c3f 100644 --- a/tests/example-provider/vagrant/vagrant.config +++ b/tests/example-provider/vagrant/vagrant.config @@ -18,6 +18,4 @@ NODE='node1' SUDO="sudo -u ${USER}" PROVIDERDIR="/home/${USER}/leap/configuration" PLATFORMDIR="/srv/leap_platform" -LEAP="$SUDO /usr/local/bin/leap" -GIT="$SUDO git" - +LEAP="$SUDO /usr/local/bin/leap $OPTS" diff --git a/tests/platform-ci/Gemfile b/tests/platform-ci/Gemfile index 44a8637d..4cf14e43 100644 --- a/tests/platform-ci/Gemfile +++ b/tests/platform-ci/Gemfile @@ -16,4 +16,5 @@ group :test do # Install fog-aws because the gem dependency of leap_cli is now optional gem "fog-aws" gem "leap_cli", :git => 'https://leap.se/git/leap_cli.git' + gem "fog-aws" end diff --git a/tests/platform-ci/README.md b/tests/platform-ci/README.md index 60c17e41..2eae5ca0 100644 --- a/tests/platform-ci/README.md +++ b/tests/platform-ci/README.md @@ -1,15 +1,53 @@ -Continuous integration tests for the leap_platform code. +# Continuous integration tests for the leap_platform code -Usage: +# Setup + cd tests/platform-ci ./setup.sh + +# Run syntax checks and test if catalog compiles + bin/rake test:syntax - bin/rake test:catalog + bin/rake catalog For a list of all tasks: bin/rake -T -To create a virtual provider, run tests on it, then tear it down: +# Full integration test + +You can create a virtual provider using AWS, run tests on it, then tear it down +when the tests succeed. +In order to do so, you need to set your AWS credentials as environment variables: + + export AWS_ACCESS_KEY='...' + export AWS_SECRET_KEY='...' + +If you want to login to this machine during or after the deploy you need to + + export SSH_PRIVATE_KEY=$(cat ~/.ssh/id_rsa) + +then start the deply test with + + ./ci-build.sh + +# Running tests with docker and gitlab-runner + +Another possibility to run the platform tests is to use [gitlab-runner](https://docs.gitlab.com/runner/) +together with [Docker](https://www.docker.com/). + +Export `AWS_ACCESS_KEY`, `AWS_SECRET_KEY` and `SSH_PRIVATE_KEY` as shown above. +From the root dir of this repo run: + + gitlab-runner exec docker --env AWS_ACCESS_KEY="$AWS_ACCESS_KEY" --env AWS_SECRET_KEY="$AWS_SECRET_KEY" --env platform_PROVIDER_SSH_PRIVATE_KEY="$SSH_PRIVATE_KEY" deploy_test + +See `.gitlab-ci.yml` for all the different test jobs. + +To ssh into the VM you first need to enter the docker container: + + docker exec -u cirunner -it $(docker ps --latest -q) bash + +From there you can access the test provider config directory and ssh into the VM: - ./ci-build.sh + cd /builds/project-0/tests/platform-ci/provider/ + leap ssh citest0 diff --git a/tests/platform-ci/ci-build.sh b/tests/platform-ci/ci-build.sh index be492d32..9bdf75fb 100755 --- a/tests/platform-ci/ci-build.sh +++ b/tests/platform-ci/ci-build.sh @@ -12,9 +12,9 @@ # * AWS credentials as environment variables: # * `AWS_ACCESS_KEY` # * `AWS_SECRET_KEY` -# * ssh private key used to login to remove vm -# * `SSH_PRIVATE_KEY` -# +# * ssh private keys used to clone providers: +# * `bitmask_PROVIDER_SSH_PRIVATE_KEY` +# * `ibex_PROVIDER_SSH_PRIVATE_KEY` # exit if any commands returns non-zero status set -e @@ -39,6 +39,9 @@ ROOTDIR=$(readlink -f "$(dirname $0)") # leap_platform PLATFORMDIR=$(readlink -f "${ROOTDIR}/../..") +# deb repo component to configure +COMPONENT=${COMPONENT:-"master"} + # In the gitlab CI pipeline leap is installed in a different # stage by bundle. To debug you can run a single CI job locally # so we install leap_cli as gem here. @@ -63,22 +66,50 @@ deploy() { LEAP_CMD deploy "$TAG" } +leap_info() { + echo "Running leap info on $TAG" + LEAP_CMD info "${TAG}" +} + test() { LEAP_CMD test "$TAG" } +ssh_setup() { + # set the provider name from the first argument passed to the function + provider_name=$1 + # set CI_SSH_SECRET_PRIVATE_KEY to the variable name keyed off of the provider_name + CI_SSH_SECRET_PRIVATE_KEY=${provider_name}_PROVIDER_SSH_PRIVATE_KEY + # Set the SSH_PRIVATE_KEY to the value provided in the CI runner secret variable setting in gitlab + SSH_PRIVATE_KEY=${!CI_SSH_SECRET_PRIVATE_KEY} + echo "Working with provider: $provider_name" + [ -z "$SSH_PRIVATE_KEY" ] && fail "${provider_name}_PROVIDER_SSH_PRIVATE_KEY is not set - please provide it as env variable." + # Configure ssh keypair + [ -d ~/.ssh ] || /bin/mkdir ~/.ssh + /bin/echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa + /bin/chmod 600 ~/.ssh/id_rsa + /bin/cp "${ROOTDIR}/provider/users/gitlab-runner-${provider_name}/gitlab-runner-${provider_name}_ssh.pub" ~/.ssh/id_rsa.pub +} + build_from_scratch() { - # when using gitlab-runner locally, CI_BUILD_ID is always 1 which + # setup ssh keys + ssh_setup platform + + # allow passing into the function the services, use a default set if empty + SERVICES=$1 + if [ -z "$SERVICES" ] + then + SERVICES='couchdb,soledad,mx,webapp,tor_relay,monitor' + fi + + # when using gitlab-runner locally, CI_JOB_ID is always 1 which # will conflict with running/terminating AWS instances in subsequent runs # therefore we pick a random number in this case [ "${CI_JOB_ID}" == "1" ] && CI_JOB_ID="000${RANDOM}" # create node(s) with unique id so we can run tests in parallel NAME="citest${CI_JOB_ID:-0}" - - TAG='single' - SERVICES='couchdb,soledad,mx,webapp,tor,monitor' # leap_platform/tests/platform-ci/provider PROVIDERDIR="${ROOTDIR}/provider" @@ -88,12 +119,11 @@ build_from_scratch() { # Create cloud.json needed for `leap vm` commands using AWS credentials which jq || ( apt-get update -y && apt-get install jq -y ) - # Dsiable xtrace + # Disable xtrace set +x [ -z "$AWS_ACCESS_KEY" ] && fail "\$AWS_ACCESS_KEY is not set - please provide it as env variable." [ -z "$AWS_SECRET_KEY" ] && fail "\$AWS_SECRET_KEY is not set - please provide it as env variable." - [ -z "$SSH_PRIVATE_KEY" ] && fail "\$SSH_PRIVATE_KEY is not set - please provide it as env variable." /usr/bin/jq ".platform_ci.auth |= .+ {\"aws_access_key_id\":\"$AWS_ACCESS_KEY\", \"aws_secret_access_key\":\"$AWS_SECRET_KEY\"}" < cloud.json.template > cloud.json # Enable xtrace again only if it was set at beginning of script @@ -102,7 +132,8 @@ build_from_scratch() { [ -d "./tags" ] || mkdir "./tags" /bin/echo "{\"environment\": \"$TAG\"}" | /usr/bin/json_pp > "${PROVIDERDIR}/tags/${TAG}.json" - pwd + # configure deb repo component + echo '{}' | jq ".sources.platform.apt |= { \"source\": \"http://deb.leap.se/platform\", \"component\": \"${COMPONENT}\" }" > common.json # remove old cached nodes echo "Removing old cached nodes..." @@ -123,53 +154,126 @@ build_from_scratch() { echo "Running leap node init on TAG: $TAG" LEAP_CMD node init "$TAG" - echo "Running leap info on $TAG" - LEAP_CMD info "${TAG}" } run() { - echo "Cloning $1 repo: $2" - git clone -q --depth 1 "$2" - cd "$1" - git rev-parse HEAD - echo -n "Operating in the $1 directory: " - pwd - echo "Listing current node information..." - LEAP_CMD list - echo "Attempting a deploy..." - deploy - echo "Attempting to run tests..." - test + provider_name=$1 + provider_URI=$2 + platform_branch=$3 + + # setup ssh keys + ssh_setup "$provider_name" + + # Setup the provider repository + echo "Setting up the provider repository: $provider_name by cloning $provider_URI" + git clone -q --depth 1 "$provider_URI" + cd "$provider_name" + echo -n "$provider_name repo at revision: " + git rev-parse HEAD + echo -n "Operating in the $provider_name directory: " + pwd + + + # If the third argument is set make sure we are on that platform branch + if [[ -n $platform_branch ]] + then + echo "Checking out $platform_branch branch of platform" + cd "$PLATFORMDIR" + git checkout -B "$platform_branch" + fi + + cd "${ROOTDIR}/${provider_name}" + echo "Listing current node information..." + LEAP_CMD list + + # Do the deployment + echo "Attempting a deploy..." + LEAP_CMD cert renew "$CI_JOB_NAME" + deploy + echo "Attempting to run tests..." + test +} + +soledad_migration() { + # check the version of soledad installed + # if the version is not greater than 0.9, we need to do the migration + if ! LEAP_CMD run "dpkg --compare-versions \$(dpkg -l |grep soledad-server|grep ^ii|awk '{ print \$3}') gt 0.8" vm |grep -q oops + then + echo "Need to migrate from soledad 0.8!" + if ! LEAP_CMD run 'systemctl stop leap-mx' vm + then fail + fi + if ! LEAP_CMD run 'systemctl stop soledad-server' vm + then fail + fi + if ! LEAP_CMD run --stream '/usr/share/soledad-server/migration/0.9/migrate.py --log-file /dev/stdout --verbose --do-migrate | tee /var/log/leap/soledad_migration.log' vm + then fail + fi + if ! LEAP_CMD run 'systemctl start leap-mx' vm + then fail + fi + if ! LEAP_CMD run 'systemctl start soledad-server' vm + then fail + fi + fi } upgrade_test() { - # Checkout stable branch containing last release + # Checkout stable branch containing previous stable release # and deploy this cd "$PLATFORMDIR" - git remote add leap https://leap.se/git/leap_platform + # due to cache, this remote is sometimes already added + git remote add leap https://leap.se/git/leap_platform || true git fetch leap - git checkout -b leap_stable remotes/leap/stable + echo "Checking out leap/stable" + git checkout -b leap_stable remotes/leap/stable || true + echo -n "Current version: " + git rev-parse HEAD # After checking out a different platform branch # bundle install is needed again cd "$ROOTDIR" /usr/local/bin/bundle install cd "$PROVIDERDIR" - - build_from_scratch + LEAP_CMD --version + build_from_scratch 'couchdb,soledad,mx,webapp,tor,monitor' deploy + leap_info + # In 0.9 leap info did not output apt sources, so we do it manually + # but can remove it for next release + cat /etc/apt/sources.list.d/* test # Checkout HEAD of current branch and re-deploy cd "$PLATFORMDIR" - git checkout "$CI_COMMIT_REF" + echo "Checking out: $CI_COMMIT_SHA" + git checkout "$CI_COMMIT_SHA" + echo -n "Current version: " + git rev-parse HEAD # After checking out a different platform branch # bundle install is needed again cd "$ROOTDIR" /usr/local/bin/bundle install cd "$PROVIDERDIR" + LEAP_CMD --version + + # due to the 'tor' service no longer being valid in 0.10, we need to change + # that service to 'tor_relay'. This is done by changing the services array + # with jq to be set to the full correct list of services + jq '.services = ["couchdb","soledad","mx","webapp","tor_relay","monitor"]' < nodes/${NAME}.json deploy + + # pre-migration test + # allowed to fail because when a migration is needed, soledad-server refuses to start + test || /bin/true + + # check for soledad migration, and run it if necessary + soledad_migration + + leap_info + + # run the test again, this should succeed test cleanup @@ -192,12 +296,6 @@ cleanup() { # Ensure we don't output secret stuff to console even when running in verbose mode with -x set +x -# Configure ssh keypair -[ -d ~/.ssh ] || /bin/mkdir ~/.ssh -/bin/echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa -/bin/chmod 600 ~/.ssh/id_rsa -/bin/cp "${ROOTDIR}/provider/users/gitlab-runner/gitlab-runner_ssh.pub" ~/.ssh/id_rsa.pub - # Enable xtrace again only if it was set at beginning of script [[ $xtrace == true ]] && set -x @@ -208,15 +306,16 @@ case "$CI_JOB_NAME" in ;; mail.bitmask.net) TAG='demomail' - run bitmask ssh://gitolite@leap.se/bitmask + run bitmask ssh://gitolite@leap.se/bitmask master ;; demo.bitmask.net) TAG='demovpn' - run bitmask ssh://gitolite@leap.se/bitmask + run bitmask ssh://gitolite@leap.se/bitmask master ;; deploy_test*) build_from_scratch deploy + leap_info test cleanup ;; diff --git a/tests/platform-ci/provider/nodes/catalogtest.json b/tests/platform-ci/provider/nodes/catalogtest.json index 05703666..bbf79d9e 100644 --- a/tests/platform-ci/provider/nodes/catalogtest.json +++ b/tests/platform-ci/provider/nodes/catalogtest.json @@ -10,7 +10,7 @@ "webapp", "monitor", "openvpn", - "tor", + "tor_relay", "obfsproxy", "static" ], diff --git a/tests/platform-ci/provider/provider.json b/tests/platform-ci/provider/provider.json index 218ff529..687f662a 100644 --- a/tests/platform-ci/provider/provider.json +++ b/tests/platform-ci/provider/provider.json @@ -10,7 +10,7 @@ "en": "You really should change this text" }, "contacts": { - "default": "root@example.org" + "default": "sysdevs@leap.se" }, "languages": ["en"], "default_language": "en", diff --git a/tests/platform-ci/provider/users/gitlab-runner/gitlab-runner_ssh.pub b/tests/platform-ci/provider/users/gitlab-runner/gitlab-runner_ssh.pub deleted file mode 100644 index 3e72b70f..00000000 --- a/tests/platform-ci/provider/users/gitlab-runner/gitlab-runner_ssh.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDEtniDgIYEm4WtGgiQsZKBpY8x3tbzDBIoMLbZT496juCu4c3f+F5KkMPLmYRPcAupF8tVf+j7Fns7z69PuTjdGfe/cA9CTw/4sNAu3iLpunGR0d2Wtctez5mwz13bKRu9fck3H9p2F9Z47vMKtRTJJ6iIgaUVWU/eFd/MSMJeUVd2ns4Wr7SkHCBB3PV+QL1xl4+AZsUtnGVQ5cE4MZZFia/g6SlrKQYFtLRVIIpDuuaDSvULg1BFMhSCBDNygts8dKTJsCEQYeGVvHZaDwtKTnMqEIwBP4TkIoP+YWnZTPrGywFEJOlZ8b+4HdgdUAFLcFCycWMM9nVcWX7P2lIN gitlab-runner_ssh diff --git a/tests/platform-ci/setup.sh b/tests/platform-ci/setup.sh index e92dddc7..c10cb4c8 100755 --- a/tests/platform-ci/setup.sh +++ b/tests/platform-ci/setup.sh @@ -1,5 +1,5 @@ #!/bin/sh which bundle || /usr/bin/apt install bundle -/usr/local/bin/bundle install --binstubs --path=vendor --with=test --jobs "$(nproc)" -/usr/local/bin/bundle exec leap -v2 --yes help +bundle install --binstubs --path=vendor --with=test --jobs "$(nproc)" +bundle exec leap -v2 --yes help diff --git a/tests/server-tests/helpers/couchdb_helper.rb b/tests/server-tests/helpers/couchdb_helper.rb index efb2c2bf..0b6671ee 100644 --- a/tests/server-tests/helpers/couchdb_helper.rb +++ b/tests/server-tests/helpers/couchdb_helper.rb @@ -31,35 +31,6 @@ class LeapTest end # - # generates a couchdb url for accessing couchdb via haproxy - # - # example properties: - # - # haproxy: - # couch: - # listen_port: 4096 - # servers: - # panda: - # backup: false - # host: localhost - # port: 4000 - # weight: 100 - # writable: true - # - def couchdb_url_via_haproxy(path="", options=nil) - path = path.gsub('"', '%22') - if options && options[:username] && options[:password] - userpart = "%{username}:%{password}@" % options - else - userpart = "" - end - port = assert_property('haproxy.couch.listen_port') - return URLString.new("http://#{userpart}localhost:#{port}#{path}").tap { |url| - url.memo = '(via haproxy)' - } - end - - # # generates a couchdb url for when couchdb is running locally. # # example properties: @@ -140,4 +111,4 @@ class LeapTest end end -end
\ No newline at end of file +end diff --git a/tests/server-tests/helpers/http_helper.rb b/tests/server-tests/helpers/http_helper.rb index 0d0bb7d5..3a1df9e7 100644 --- a/tests/server-tests/helpers/http_helper.rb +++ b/tests/server-tests/helpers/http_helper.rb @@ -5,7 +5,7 @@ class LeapTest # # In order to easily provide detailed error messages, it is useful # to append a memo to a url string that details what this url is for - # (e.g. stunnel, haproxy, etc). + # (e.g. stunnel, etc). # # So, the url happens to be a UrlString, the memo field is used # if there is an error in assert_get. @@ -154,4 +154,4 @@ class LeapTest request end -end
\ No newline at end of file +end diff --git a/tests/server-tests/helpers/os_helper.rb b/tests/server-tests/helpers/os_helper.rb index 9923d5b1..6a71388c 100644 --- a/tests/server-tests/helpers/os_helper.rb +++ b/tests/server-tests/helpers/os_helper.rb @@ -20,11 +20,29 @@ class LeapTest }.compact end - def assert_running(process, options={}) - processes = pgrep(process) - assert processes.any?, "No running process for #{process}" - if options[:single] - assert processes.length == 1, "More than one process for #{process}" + # + # passes if the specified process is runnin. + # + # arguments: + # + # match => VALUE -- scan process table for VALUE + # service => VALUE -- call systemctl is-active VALUE + # + # single => true|false -- if true, there must be one result + # + def assert_running(match:nil, service:nil, single:false) + if match + processes = pgrep(match) + assert processes.any?, "No running process for #{match}" + if single + assert processes.length == 1, "More than one process for #{match}" + end + elsif service + `systemctl is-active #{service} 2>&1` + if $?.exitstatus != 0 + output = `systemctl status #{service} 2>&1` + fail "Service '#{service}' is not running:\n#{output}" + end end end diff --git a/tests/server-tests/helpers/soledad_sync.py b/tests/server-tests/helpers/soledad_sync.py index f4fc81ae..a92ec68f 100755 --- a/tests/server-tests/helpers/soledad_sync.py +++ b/tests/server-tests/helpers/soledad_sync.py @@ -27,6 +27,7 @@ os.environ['SKIP_TWISTED_SSL_CHECK'] = '1' from twisted.internet import defer, reactor from twisted.python import log +from twisted.python.lockfile import FilesystemLock from client_side_db import get_soledad_instance from leap.common.events import flags @@ -35,6 +36,7 @@ flags.set_events_enabled(False) NUMDOCS = 1 USAGE = "Usage: %s uuid token server cert_file password" % sys.argv[0] +SYNC_TIMEOUT = 60 def bail(msg, exitcode): @@ -42,6 +44,13 @@ def bail(msg, exitcode): sys.exit(exitcode) +def obtain_lock(): + scriptname = os.path.basename(__file__) + lockfile = os.path.join(tempfile.gettempdir(), scriptname + '.lock') + lock = FilesystemLock(lockfile) + return lock.lock() + + def create_docs(soledad): """ Populates the soledad database with dummy messages, so we can exercise @@ -64,16 +73,30 @@ if __name__ == '__main__': if len(sys.argv) < 6: bail(USAGE, 2) + if not obtain_lock(): + bail("another instance is already running", 1) + uuid, token, server, cert_file, passphrase = sys.argv[1:] s = get_soledad_instance( uuid, passphrase, tempdir, server, cert_file, token) + def syncWithTimeout(_): + d = s.sync() + reactor.callLater(SYNC_TIMEOUT, d.cancel) + return d + def onSyncDone(sync_result): print "SYNC_RESULT:", sync_result s.close() rm_tempdir() reactor.stop() + def trap_cancel(f): + f.trap(defer.CancelledError) + log.err("sync timed out after %s seconds" % SYNC_TIMEOUT) + rm_tempdir() + reactor.stop() + def log_and_exit(f): log.err(f) rm_tempdir() @@ -81,8 +104,9 @@ if __name__ == '__main__': def start_sync(): d = create_docs(s) - d.addCallback(lambda _: s.sync()) + d.addCallback(syncWithTimeout) d.addCallback(onSyncDone) + d.addErrback(trap_cancel) d.addErrback(log_and_exit) reactor.callWhenRunning(start_sync) diff --git a/tests/server-tests/white-box/couchdb.rb b/tests/server-tests/white-box/couchdb.rb index 44a2769b..dcf71bc7 100644 --- a/tests/server-tests/white-box/couchdb.rb +++ b/tests/server-tests/white-box/couchdb.rb @@ -9,9 +9,9 @@ class CouchDB < LeapTest end def test_00_Are_daemons_running? - assert_running 'bin/beam' + assert_running match: 'bin/beam' if multimaster? - assert_running 'bin/epmd' + assert_running match: 'bin/epmd' end pass end diff --git a/tests/server-tests/white-box/mx.rb b/tests/server-tests/white-box/mx.rb index ecc8686c..dfad0eed 100644 --- a/tests/server-tests/white-box/mx.rb +++ b/tests/server-tests/white-box/mx.rb @@ -24,16 +24,6 @@ class Mx < LeapTest pass end - def test_02_Can_contact_couchdb_via_haproxy? - if property('haproxy.couch') - url = couchdb_url_via_haproxy("", couch_url_options) - assert_get(url) do |body| - assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." - end - pass - end - end - # # this test picks a random identity document, then queries # using the by_address view for that same document again. @@ -52,17 +42,17 @@ class Mx < LeapTest end def test_04_Are_MX_daemons_running? - assert_running '.*/usr/bin/twistd.*mx.tac' - assert_running '^/usr/lib/postfix/master$' - assert_running '^/usr/sbin/postfwd' - assert_running 'postfwd2::cache$' - assert_running 'postfwd2::policy$' - assert_running '^/usr/sbin/unbound' - assert_running '^/usr/bin/freshclam' - assert_running '^/usr/sbin/opendkim' + assert_running match: '.*/usr/bin/twistd.*mx.tac' + assert_running match: '^/usr/lib/postfix/master$' + assert_running match: '^/usr/sbin/postfwd' + assert_running match: 'postfwd2::cache$' + assert_running match: 'postfwd2::policy$' + assert_running match: '^/usr/sbin/unbound' + assert_running match: '^/usr/bin/freshclam' + assert_running match: '^/usr/sbin/opendkim' if Dir.glob("/var/lib/clamav/main.{c[vl]d,inc}").size > 0 and Dir.glob("/var/lib/clamav/daily.{c[vl]d,inc}").size > 0 - assert_running '^/usr/sbin/clamd' - assert_running '^/usr/sbin/clamav-milter' + assert_running match: '^/usr/sbin/clamd' + assert_running match: '^/usr/sbin/clamav-milter' pass else skip "Downloading the clamav signature files (/var/lib/clamav/{daily,main}.{c[vl]d,inc}) is still in progress, so clamd is not running." diff --git a/tests/server-tests/white-box/openvpn.rb b/tests/server-tests/white-box/openvpn.rb index d5cc2265..adda34a9 100644 --- a/tests/server-tests/white-box/openvpn.rb +++ b/tests/server-tests/white-box/openvpn.rb @@ -7,10 +7,46 @@ class OpenVPN < LeapTest end def test_01_Are_daemons_running? - assert_running '^/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf$' - assert_running '^/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf$' - assert_running '^/usr/sbin/unbound' + assert_running match: '^/usr/sbin/openvpn .* /etc/openvpn/tcp_config.conf$' + assert_running match: '^/usr/sbin/openvpn .* /etc/openvpn/udp_config.conf$' + assert_running match: '^/usr/sbin/unbound' pass end + def test_02_Can_connect_to_openvpn? + # because of the way the firewall rules are currently set up, you can only + # connect to the standard 1194 openvpn port when you are connecting + # from the same host as openvpn is running on. + # + # so, this is disabled for now: + # $node['openvpn']['ports'].each {|port| ...} + # + + $node['openvpn']['protocols'].each do |protocol| + assert_openvpn_is_bound_to_port($node['openvpn']['gateway_address'], protocol, 1194) + end + pass + end + + private + + # + # asserting succeeds if openvpn appears to be correctly bound and we can + # connect to it. we don't actually try to establish a vpn connection in this + # test, we just check to see that it sort of looks like it is openvpn running + # on the port. + # + def assert_openvpn_is_bound_to_port(ip_address, protocol, port) + protocol = protocol.downcase + if protocol == 'udp' + # this sends a magic string to openvpn to attempt to start the protocol. + nc_output = `/bin/echo -e "\\x38\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00" | timeout 0.5 nc -u #{ip_address} #{port}`.strip + assert !nc_output.empty?, "Could not connect to OpenVPN daemon at #{ip_address} on port #{port} (#{protocol})." + elsif protocol == 'tcp' + assert system("openssl s_client -connect #{ip_address}:#{port} 2>&1 | grep -q CONNECTED"), + "Could not connect to OpenVPN daemon at #{ip_address} on port #{port} (#{protocol})." + else + assert false, "invalid openvpn protocol #{protocol}" + end + end end diff --git a/tests/server-tests/white-box/soledad.rb b/tests/server-tests/white-box/soledad.rb index 7c6918f9..112d6b9b 100644 --- a/tests/server-tests/white-box/soledad.rb +++ b/tests/server-tests/white-box/soledad.rb @@ -10,7 +10,7 @@ class Soledad < LeapTest end def test_00_Is_Soledad_running? - assert_running '/usr/bin/python /usr/bin/twistd --uid=soledad --gid=soledad --pidfile=/var/run/soledad.pid.*' + assert_running service: 'soledad-server' pass end diff --git a/tests/server-tests/white-box/webapp.rb b/tests/server-tests/white-box/webapp.rb index da1ec8c5..42adf219 100644 --- a/tests/server-tests/white-box/webapp.rb +++ b/tests/server-tests/white-box/webapp.rb @@ -4,6 +4,7 @@ require 'json' class Webapp < LeapTest depends_on "Network" + depends_on "Soledad" if service?(:soledad) def setup end @@ -16,19 +17,9 @@ class Webapp < LeapTest pass end - def test_02_Can_contact_couchdb_via_haproxy? - if property('haproxy.couch') - url = couchdb_url_via_haproxy("", url_options) - assert_get(url) do |body| - assert_match /"couchdb":"Welcome"/, body, "Request to #{url} should return couchdb welcome message." - end - pass - end - end - def test_03_Are_daemons_running? - assert_running '^/usr/sbin/apache2' - assert_running '^ruby /usr/bin/nickserver' + assert_running match: '^/usr/sbin/apache2' + assert_running match: 'ruby /usr/bin/nickserver' pass end |