Merge remote-tracking branch 'refs/remotes/ivan/feature/bundling-automation' into develop

4 files changed, 280 insertions, 132 deletions

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..258a61e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,45 @@
+# Bundler
+
+This application is intended to create [Bitmask](https://bitmask.net)
+[standalone bundles](https://bitmask.net/en/install/linux#install-stand-alone-bundle).
+This should eventually become the main tool in order to create reproducible
+builds.
+
+## What do you need
+
+* Two virtual machines:
+  * Debian 7.1 32bits - for the 32bits bundle
+  * Debian 7.1 64bits - for the 64bits bundle
+* two scripts in this repository,
+* an internet connection,
+* approximately 1.3Gb of disk space,
+* (optional) configure `sudo` for your non-root user,
+* patience.
+
+The script uses `sudo` which is installed by default on Debian, but you need to
+configure your non-root user to be able to use `sudo`.
+
+Edit the `/etc/sudoers` file and add this line at the bottom:
+`leap ALL=NOPASSWD: ALL`
+
+*HEADS UP*: That line allows the user 'leap' to use `sudo` without being asked
+for a password, that makes the script easier to use but it would be a security
+problem. If you use this script in a VM and only for bundling purposes then it
+shouldn't be a problem.
+
+
+
+
+## How to use
+
+You need to copy the scripts `createbundle.sh` and `copy-binaries.sh` to a VM
+and run `./createbundle.sh`, after that it should be all automagically
+processed.
+
+You can start the script with the parameter `nightly` to build a bundle from
+the latest `develop` code, otherwise it will bundle from the latest tag.
+
+The resulting bundle will be saved in:
+`/home/leap/bitmask.bundle/bundle.output/` under some name like
+`Bitmask-linux64-2014-09-24-9b3b7f6f.tar.bz2` in case of bundling a *nightly*
+release, or `Bitmask-linux64-0.7.0.tar.bz2` in case of a *normal* release.
diff --git a/README.rst b/README.rst
deleted file mode 100644
index 734d82b..0000000
--- a/README.rst
+++ /dev/null
@@ -1,132 +0,0 @@
-Bundler
-+++++++
-
-This application is intended to create bundles for the platform in which its being run. This should eventually become the main tool in order to create reproducible builds.
-
-How to use
-++++++++++
-
-NOTE: Most of this will be done automatically in a while, but this is how it's done now.
-
-- Install Xcode and command line tools (OSX only)
-- Create a new virtualenv
-
-::
-  mkvirtualenv bundle
-
-- Install bundler deps (OSX or Linux)
-
-::
-  pip install -r pkg/requirements.pip
-
-- Install bundler deps (Windows)
-
-::
-  pip install pbs  # There's no sh in Windows
-  pip install modulegraph altgraph
-
-- Install the needed dependencies
-
-::
-  # Linux
-  aptitude install libsqlite3-dev
-
-  # In Windows you'll need sqlite3 in your path
-
-- psutils is a dependency for another dependency, it might get installed in a zip form, which we don't want, so we install it by hand for now
-
-::
-  pip install psutil
-
-- We need a slightly different python-gnupg, so clone from a different repo
-
-::
-  git clone https://github.com/chiiph/python-gnupg
-  cd python-gnupg/
-  git checkout develop
-  git pull origin develop
-  python setup.py develop
-
-- Same thing with protobuf.socketrpc
-
-::
-  git clone https://github.com/chiiph/protobuf-socket-rpc
-  cd protobuf-socket-rpc
-  python setup.py easy_install -Z .
-
-- Install Qt 4.8 in whatever way you prefer.
-
-- Build PySide:
-
-::
-  git clone git://gitorious.org/pyside/apiextractor.git
-  git clone git://gitorious.org/pyside/generatorrunner.git
-  git clone git://gitorious.org/pyside/shiboken.git
-  git clone git://gitorious.org/pyside/pyside.git
-  git clone git://gitorious.org/pyside/pyside-tools.git
-
-  # OSX
-  export PYSIDESANDBOXPATH=$HOME/Code/pyside/sandbox
-  export DYLD_LIBRARY_PATH=$PYSIDESANDBOXPATH/lib:$DYLD_LIBRARY_PATH
-
-  # Linux
-  export PYSIDESANDBOXPATH=$HOME/sandbox
-  export LD_LIBRARY_PATH=$PYSIDESANDBOXPATH/lib:$LD_LIBRARY_PATH
-
-  export PATH=$PYSIDESANDBOXPATH/bin:$PATH
-  export PYTHONPATH=$PYSIDESANDBOXPATH/lib/python2.6/site-packages:$PYTHONPATH
-  export PKG_CONFIG_PATH=$PYSIDESANDBOXPATH/lib/pkgconfig:$PKG_CONFIG_PATH
-
-  alias runcmake='cmake .. -DCMAKE_INSTALL_PREFIX=$PYSIDESANDBOXPATH'
-
-  # In OSX, the paths may vary depending on the Qt installation
-  runcmake -DCMAKE_OSX_DEPLOYMENT_TARGET=10.7 -DCMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.8.sdk .. -DQT_QMAKE_EXECUTABLE=/usr/local/bin/qmake -DQT_INCLUDE_DIR=/usr/local/include/ -DQT_INCLUDES=/usr/local/include/ -DALTERNATIVE_QT_INCLUDE_DIR=/usr/local/include/
-  # In Linux
-  runcmake ..
-
-  make install
-
-  # Make them available from the virtualenv
-  ln -s $PYSIDESANDBOXPATH/lib/python2.7/site-packages/PySide $VIRTUAL_ENV/lib/python2.7/site-packages/PySide
-  ln -s $PYSIDESANDBOXPATH/lib/python2.7/site-packages/pysideuic $VIRTUAL_ENV/lib/python2.7/site-packages/pysideuic
-
-- Create a paths file: The problem is that inside a virtualenv we don't have access to the real distutils, so we'll need to look for it on the "original" (i.e. non-virtualenv) paths for python.
-
-::
-  python bundler/create_paths.py <paths file>
-
-- Collect the binaries. We aren't building everything yet, so you'll need to collect the following files:
-
-::
-  # OSX
-  Bitmask <-- this is the bitmask_launcher
-  Python
-  QtCore
-  QtGui
-  cocoasudo
-  gpg
-  libboost_filesystem.dylib
-  libboost_python.dylib
-  libboost_system.dylib
-  libpng15.15.dylib
-  libpyside-python2.7.1.2.dylib
-  libshiboken-python2.7.1.2.dylib
-  openvpn.files
-  openvpn.leap
-  qt_menu.nib
-  tuntap-installer.app
-
-- (Optional) Seed a configuration: You might want to create a bundle with a specific configuration pinned providers.
-
-- Create the bundle:
-
-::
-  # On OSX and Linux
-  python bundler/main.py --workon <path/to/bundle/temp> --paths-file <paths file> --binaries <binaries dir> --seeded-config <seeded config> [--nightly] --do gitclone pythonsetup
-  python bundler/main.py --workon <path/to/bundle/temp> --paths-file <paths file> --binaries <binaries dir> --seeded-config <seeded config> [--nightly] --skip gitclone pythonsetup
-
-  # On Windows
-  python bundler/main.py --workon <path/to/bundle/temp> --paths-file <paths file> --binaries <binaries dir> --seeded-config <seeded config> [--nightly] --do gitclone
-  # Modify bitmask_client/pkg/requirements.pip, comment out python-daemon
-  python bundler/main.py --workon <path/to/bundle/temp> --paths-file <paths file> --binaries <binaries dir> --seeded-config <seeded config> [--nightly] --do pythonsetup
-  python bundler/main.py --workon <path/to/bundle/temp> --paths-file <paths file> --binaries <binaries dir> --seeded-config <seeded config> [--nightly] --skip gitclone pythonsetup
diff --git a/copy-binaries.sh b/copy-binaries.sh
new file mode 100755
index 0000000..1d8b809
--- /dev/null
+++ b/copy-binaries.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -e  # Exit immediately if a command exits with a non-zero status.
+
+BASE='/home/leap/bitmask.bundle'
+BOOST="$BASE/boost_1_56_0"
+
+# Note: we could use:
+# ARCH=`uname -i`
+# but it does not work on a VM (tested in i386 returns unknown)
+if [[ `getconf LONG_BIT` == "64" ]]; then
+    ARCH='x86_64-linux-gnu'
+else
+    ARCH='i386-linux-gnu'
+fi
+
+cd $BASE
+mkdir binaries && cd binaries
+
+cp /usr/bin/gpg .
+cp $BASE/bitmask_launcher/build/src/launcher bitmask
+cp $BOOST/stage/lib/libboost_filesystem.so.1.56.0 .
+cp $BOOST/stage/lib/libboost_python.so.1.56.0 .
+cp $BOOST/stage/lib/libboost_system.so.1.56.0 .
+
+cp $BASE/pyside-setup.git/pyside_package/PySide/libpyside-python2.7.so.1.2 .
+cp $BASE/pyside-setup.git/pyside_package/PySide/libshiboken-python2.7.so.1.2 .
+
+cp /usr/lib/$ARCH/libQtGui.so libQtGui.non-ubuntu
+cp /usr/lib/$ARCH/libQtCore.so libQtCore.non-ubuntu
+
+cp /usr/lib/$ARCH/libaudio.so.2 .
+cp /usr/lib/$ARCH/libffi.so.5 .
+cp /usr/lib/$ARCH/libfontconfig.so.1 .
+cp /lib/$ARCH/libpng12.so.0 .  # NOTE: it should be libpng15.so.15
+cp /usr/lib/libpython2.7.so.1.0 .
+cp /usr/lib/$ARCH/libssl.so.1.0.0 .
+cp /usr/lib/$ARCH/libstdc++.so.6 .
+
+touch root.json  # empty file for TUF
+
+mkdir openvpn.files
+cd openvpn.files
+cp $BASE/openvpn/src/openvpn/openvpn leap-openvpn
+
+# TODO: to avoid network requests this should be copied from the cloned repositories
+# after `bundler gitclone` and before `bundler pythonsetup`
+wget https://raw.githubusercontent.com/leapcode/bitmask_client/develop/pkg/linux/bitmask-root
+wget https://raw.githubusercontent.com/leapcode/bitmask_client/develop/pkg/linux/leap-install-helper.sh
+wget https://raw.githubusercontent.com/leapcode/bitmask_client/develop/pkg/linux/polkit/se.leap.bitmask.bundle.policy
+chmod +x bitmask-root
+chmod +x leap-install-helper.sh
diff --git a/createbundle.sh b/createbundle.sh
new file mode 100755
index 0000000..9856d2a
--- /dev/null
+++ b/createbundle.sh
@@ -0,0 +1,183 @@
+#!/bin/bash
+######################################################################
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+######################################################################
+
+# This script is meant to be used as a bootstrapper for the environment to
+# create a Bitmask bundle for Linux.
+
+# This is tested to work on Debian 7.1 32 and 64 bits Virtual Machines
+
+# This script uses `sudo` which is installed by default on debian, but you need
+# to configure your non-root user to be able to use `sudo`.
+
+# Edit the /etc/sudoers file and add this line at the bottom:
+# leap ALL=NOPASSWD: ALL
+
+# WARNING: That line allows the user 'leap' to use `sudo` without being asked
+# for a password, that makes the script easier to use but it would be a
+# security problem. If you use this script in a VM and only for bundling
+# purposes then it shouldn't be a problem.
+
+# You'll need an internet connection, approximately 1.3Gb of disk space and
+# some patience.
+
+################################################################################
+
+set -e  # Exit immediately if a command exits with a non-zero status.
+
+BASE='/home/leap/bitmask.bundle'
+mkdir -p $BASE
+
+# Note: we could use:
+# ARCH=`uname -i`
+# but it does not work on a VM (tested in i386 returns unknown)
+if [[ `getconf LONG_BIT` == "64" ]]; then
+    ARCH='x86_64-linux-gnu'
+else
+    ARCH='i386-linux-gnu'
+fi
+
+install_dependencies() {
+    sudo apt-get install -y build-essential python-dev cmake git autoconf \
+        libtool liblzo2-dev libqt4-dev libxml2-dev libxslt1-dev qtmobility-dev \
+        libsqlite3-dev libffi-dev python-virtualenv
+}
+
+build_boost() {
+    cd $BASE
+
+    BOOST_NAME='boost_1_56_0'
+    wget -c http://ufpr.dl.sourceforge.net/project/boost/boost/1.56.0/$BOOST_NAME.tar.bz2
+    tar xjf $BOOST_NAME.tar.bz2
+
+    cd $BOOST_NAME/tools/build/
+    ./bootstrap.sh --with-toolset=gcc
+    ./b2 install --prefix=$BASE/boost
+    export PATH=$PATH:$BASE/boost/bin/
+
+    cd $BASE/$BOOST_NAME # boost root
+    b2 cxxflags=-std=c++0x --with-python --with-filesystem --with-system variant=release link=shared
+}
+
+build_launcher() {
+    cd $BASE
+    git clone -b develop https://leap.se/git/bitmask_launcher.git
+    cd bitmask_launcher
+
+    mkdir build
+    cd build
+
+    cmake -DBoost_INCLUDE_DIR=$BASE/$BOOST_NAME ..
+    make
+}
+
+build_openvpn() {
+    # Build openvpn to support RPATH
+    cd $BASE
+    git clone https://github.com/OpenVPN/openvpn.git
+    cd openvpn
+    autoreconf -i
+    LZO_LIBS="/usr/lib/$ARCH/liblzo2.a" ./configure LDFLAGS="-Wl,-rpath,lib/" --disable-snappy --disable-plugin-auth-pam
+    make -j2
+}
+
+build_pyside() {
+    # for more information look at:
+    # https://github.com/PySide/pyside-setup/blob/master/docs/building/linux.rst
+    cd $BASE
+    sudo pip install wheel
+
+    git clone https://github.com/PySide/pyside-setup.git pyside-setup.git
+    cd pyside-setup.git
+    python setup.py bdist_wheel --qmake=/usr/bin/qmake-qt4 --version=1.2.2
+}
+
+set_pyside_environment() {
+    arch_bits=`getconf LONG_BIT`  # '32' or '64'
+    # from https://github.com/PySide/BuildScripts/blob/master/environment.sh
+    PYSIDE="$BASE/pyside-setup.git/pyside_install/py2.7-qt4.8.2-${arch_bits}bit-release/"
+    PYTHONXY='python2.7'
+    export PATH=$PYSIDE/bin:$PATH
+    export PYTHONPATH=$PYSIDE/lib/$PYTHONXY/site-packages:$PYSIDE/lib64/$PYTHONXY/site-packages:$PYTHONPATH
+    export LD_LIBRARY_PATH=$PYSIDE/lib:$LD_LIBRARY_PATH
+    export PKG_CONFIG_PATH=$PYSIDE/lib/pkgconfig:$PKG_CONFIG_PATH
+    export DYLD_LIBRARY_PATH=$PYSIDE/lib:$DYLD_LIBRARY_PATH
+}
+
+copy_binaries() {
+    cd $BASE
+    ../copy-binaries.sh
+}
+
+create_bundler_paths() {
+    cd $BASE
+    cat > bundler.paths << EOF
+$BASE/bundler.git/bundler
+/usr/lib/python2.7
+/usr/lib/python2.7/plat-linux2
+/usr/lib/python2.7/lib-tk
+/usr/lib/python2.7/lib-old
+/usr/lib/python2.7/lib-dynload
+/usr/local/lib/python2.7/dist-packages
+/usr/lib/python2.7/dist-packages
+/usr/lib/python2.7/dist-packages/gtk-2.0
+/usr/lib/pymodules/python2.7
+EOF
+}
+
+setup_bundler() {
+    cd $BASE
+
+    git clone https://github.com/chiiph/bundler.git bundler.git
+    virtualenv bundler.venv && source bundler.venv/bin/activate
+
+    # install dependencies by hand...
+    pip install psutil
+    pip install tuf  # used in the launher, it is not in any requirements.txt
+
+    git clone https://github.com/chiiph/protobuf-socket-rpc protobuf-socket-rpc.git
+    cd protobuf-socket-rpc.git
+    python setup.py easy_install -Z .
+
+    pip install -r $BASE/bundler.git/pkg/requirements.pip
+}
+
+run_bundler() {
+    cd $BASE
+
+    # if the virtualenv is not sourced, then source it!
+    # this is helpful if you want to run this step only
+    [[ -z "$VIRTUAL_ENV"  ]] && source bundler.venv/bin/activate
+
+    set_pyside_environment
+
+    mkdir bundler.output
+    python bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths --do gitclone pythonsetup $VERSION
+    python bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths --skip gitclone pythonsetup $VERSION
+}
+
+[[ "$1" == 'nightly' ]] && VERSION='--nightly'
+
+install_dependencies
+build_boost
+build_launcher
+build_openvpn
+build_pyside
+copy_binaries
+create_bundler_paths
+setup_bundler
+run_bundler