From 08a46c32b202702bb0cd57f76be1e40f4b1de2c3 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Fri, 26 Sep 2014 18:03:05 -0300 Subject: Update README file with up to date docs. --- README.md | 45 +++++++++++++++++++++ README.rst | 132 ------------------------------------------------------------- 2 files changed, 45 insertions(+), 132 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 0000000..258a61e --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# Bundler + +This application is intended to create [Bitmask](https://bitmask.net) +[standalone bundles](https://bitmask.net/en/install/linux#install-stand-alone-bundle). +This should eventually become the main tool in order to create reproducible +builds. + +## What do you need + +* Two virtual machines: + * Debian 7.1 32bits - for the 32bits bundle + * Debian 7.1 64bits - for the 64bits bundle +* two scripts in this repository, +* an internet connection, +* approximately 1.3Gb of disk space, +* (optional) configure `sudo` for your non-root user, +* patience. + +The script uses `sudo` which is installed by default on Debian, but you need to +configure your non-root user to be able to use `sudo`. + +Edit the `/etc/sudoers` file and add this line at the bottom: +`leap ALL=NOPASSWD: ALL` + +*HEADS UP*: That line allows the user 'leap' to use `sudo` without being asked +for a password, that makes the script easier to use but it would be a security +problem. If you use this script in a VM and only for bundling purposes then it +shouldn't be a problem. + + + + +## How to use + +You need to copy the scripts `createbundle.sh` and `copy-binaries.sh` to a VM +and run `./createbundle.sh`, after that it should be all automagically +processed. + +You can start the script with the parameter `nightly` to build a bundle from +the latest `develop` code, otherwise it will bundle from the latest tag. + +The resulting bundle will be saved in: +`/home/leap/bitmask.bundle/bundle.output/` under some name like +`Bitmask-linux64-2014-09-24-9b3b7f6f.tar.bz2` in case of bundling a *nightly* +release, or `Bitmask-linux64-0.7.0.tar.bz2` in case of a *normal* release. diff --git a/README.rst b/README.rst deleted file mode 100644 index 734d82b..0000000 --- a/README.rst +++ /dev/null @@ -1,132 +0,0 @@ -Bundler -+++++++ - -This application is intended to create bundles for the platform in which its being run. This should eventually become the main tool in order to create reproducible builds. - -How to use -++++++++++ - -NOTE: Most of this will be done automatically in a while, but this is how it's done now. - -- Install Xcode and command line tools (OSX only) -- Create a new virtualenv - -:: - mkvirtualenv bundle - -- Install bundler deps (OSX or Linux) - -:: - pip install -r pkg/requirements.pip - -- Install bundler deps (Windows) - -:: - pip install pbs # There's no sh in Windows - pip install modulegraph altgraph - -- Install the needed dependencies - -:: - # Linux - aptitude install libsqlite3-dev - - # In Windows you'll need sqlite3 in your path - -- psutils is a dependency for another dependency, it might get installed in a zip form, which we don't want, so we install it by hand for now - -:: - pip install psutil - -- We need a slightly different python-gnupg, so clone from a different repo - -:: - git clone https://github.com/chiiph/python-gnupg - cd python-gnupg/ - git checkout develop - git pull origin develop - python setup.py develop - -- Same thing with protobuf.socketrpc - -:: - git clone https://github.com/chiiph/protobuf-socket-rpc - cd protobuf-socket-rpc - python setup.py easy_install -Z . - -- Install Qt 4.8 in whatever way you prefer. - -- Build PySide: - -:: - git clone git://gitorious.org/pyside/apiextractor.git - git clone git://gitorious.org/pyside/generatorrunner.git - git clone git://gitorious.org/pyside/shiboken.git - git clone git://gitorious.org/pyside/pyside.git - git clone git://gitorious.org/pyside/pyside-tools.git - - # OSX - export PYSIDESANDBOXPATH=$HOME/Code/pyside/sandbox - export DYLD_LIBRARY_PATH=$PYSIDESANDBOXPATH/lib:$DYLD_LIBRARY_PATH - - # Linux - export PYSIDESANDBOXPATH=$HOME/sandbox - export LD_LIBRARY_PATH=$PYSIDESANDBOXPATH/lib:$LD_LIBRARY_PATH - - export PATH=$PYSIDESANDBOXPATH/bin:$PATH - export PYTHONPATH=$PYSIDESANDBOXPATH/lib/python2.6/site-packages:$PYTHONPATH - export PKG_CONFIG_PATH=$PYSIDESANDBOXPATH/lib/pkgconfig:$PKG_CONFIG_PATH - - alias runcmake='cmake .. -DCMAKE_INSTALL_PREFIX=$PYSIDESANDBOXPATH' - - # In OSX, the paths may vary depending on the Qt installation - runcmake -DCMAKE_OSX_DEPLOYMENT_TARGET=10.7 -DCMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.8.sdk .. -DQT_QMAKE_EXECUTABLE=/usr/local/bin/qmake -DQT_INCLUDE_DIR=/usr/local/include/ -DQT_INCLUDES=/usr/local/include/ -DALTERNATIVE_QT_INCLUDE_DIR=/usr/local/include/ - # In Linux - runcmake .. - - make install - - # Make them available from the virtualenv - ln -s $PYSIDESANDBOXPATH/lib/python2.7/site-packages/PySide $VIRTUAL_ENV/lib/python2.7/site-packages/PySide - ln -s $PYSIDESANDBOXPATH/lib/python2.7/site-packages/pysideuic $VIRTUAL_ENV/lib/python2.7/site-packages/pysideuic - -- Create a paths file: The problem is that inside a virtualenv we don't have access to the real distutils, so we'll need to look for it on the "original" (i.e. non-virtualenv) paths for python. - -:: - python bundler/create_paths.py - -- Collect the binaries. We aren't building everything yet, so you'll need to collect the following files: - -:: - # OSX - Bitmask <-- this is the bitmask_launcher - Python - QtCore - QtGui - cocoasudo - gpg - libboost_filesystem.dylib - libboost_python.dylib - libboost_system.dylib - libpng15.15.dylib - libpyside-python2.7.1.2.dylib - libshiboken-python2.7.1.2.dylib - openvpn.files - openvpn.leap - qt_menu.nib - tuntap-installer.app - -- (Optional) Seed a configuration: You might want to create a bundle with a specific configuration pinned providers. - -- Create the bundle: - -:: - # On OSX and Linux - python bundler/main.py --workon --paths-file --binaries --seeded-config [--nightly] --do gitclone pythonsetup - python bundler/main.py --workon --paths-file --binaries --seeded-config [--nightly] --skip gitclone pythonsetup - - # On Windows - python bundler/main.py --workon --paths-file --binaries --seeded-config [--nightly] --do gitclone - # Modify bitmask_client/pkg/requirements.pip, comment out python-daemon - python bundler/main.py --workon --paths-file --binaries --seeded-config [--nightly] --do pythonsetup - python bundler/main.py --workon --paths-file --binaries --seeded-config [--nightly] --skip gitclone pythonsetup -- cgit v1.2.3 From 13822f09e17aa5a262ec69b99d5829dc69efea53 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Wed, 24 Sep 2014 16:33:09 -0300 Subject: Add automagic bundling scripts. --- copy-binaries.sh | 52 ++++++++++++++++ createbundle.sh | 183 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100755 copy-binaries.sh create mode 100755 createbundle.sh diff --git a/copy-binaries.sh b/copy-binaries.sh new file mode 100755 index 0000000..1d8b809 --- /dev/null +++ b/copy-binaries.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e # Exit immediately if a command exits with a non-zero status. + +BASE='/home/leap/bitmask.bundle' +BOOST="$BASE/boost_1_56_0" + +# Note: we could use: +# ARCH=`uname -i` +# but it does not work on a VM (tested in i386 returns unknown) +if [[ `getconf LONG_BIT` == "64" ]]; then + ARCH='x86_64-linux-gnu' +else + ARCH='i386-linux-gnu' +fi + +cd $BASE +mkdir binaries && cd binaries + +cp /usr/bin/gpg . +cp $BASE/bitmask_launcher/build/src/launcher bitmask +cp $BOOST/stage/lib/libboost_filesystem.so.1.56.0 . +cp $BOOST/stage/lib/libboost_python.so.1.56.0 . +cp $BOOST/stage/lib/libboost_system.so.1.56.0 . + +cp $BASE/pyside-setup.git/pyside_package/PySide/libpyside-python2.7.so.1.2 . +cp $BASE/pyside-setup.git/pyside_package/PySide/libshiboken-python2.7.so.1.2 . + +cp /usr/lib/$ARCH/libQtGui.so libQtGui.non-ubuntu +cp /usr/lib/$ARCH/libQtCore.so libQtCore.non-ubuntu + +cp /usr/lib/$ARCH/libaudio.so.2 . +cp /usr/lib/$ARCH/libffi.so.5 . +cp /usr/lib/$ARCH/libfontconfig.so.1 . +cp /lib/$ARCH/libpng12.so.0 . # NOTE: it should be libpng15.so.15 +cp /usr/lib/libpython2.7.so.1.0 . +cp /usr/lib/$ARCH/libssl.so.1.0.0 . +cp /usr/lib/$ARCH/libstdc++.so.6 . + +touch root.json # empty file for TUF + +mkdir openvpn.files +cd openvpn.files +cp $BASE/openvpn/src/openvpn/openvpn leap-openvpn + +# TODO: to avoid network requests this should be copied from the cloned repositories +# after `bundler gitclone` and before `bundler pythonsetup` +wget https://raw.githubusercontent.com/leapcode/bitmask_client/develop/pkg/linux/bitmask-root +wget https://raw.githubusercontent.com/leapcode/bitmask_client/develop/pkg/linux/leap-install-helper.sh +wget https://raw.githubusercontent.com/leapcode/bitmask_client/develop/pkg/linux/polkit/se.leap.bitmask.bundle.policy +chmod +x bitmask-root +chmod +x leap-install-helper.sh diff --git a/createbundle.sh b/createbundle.sh new file mode 100755 index 0000000..9856d2a --- /dev/null +++ b/createbundle.sh @@ -0,0 +1,183 @@ +#!/bin/bash +###################################################################### +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +###################################################################### + +# This script is meant to be used as a bootstrapper for the environment to +# create a Bitmask bundle for Linux. + +# This is tested to work on Debian 7.1 32 and 64 bits Virtual Machines + +# This script uses `sudo` which is installed by default on debian, but you need +# to configure your non-root user to be able to use `sudo`. + +# Edit the /etc/sudoers file and add this line at the bottom: +# leap ALL=NOPASSWD: ALL + +# WARNING: That line allows the user 'leap' to use `sudo` without being asked +# for a password, that makes the script easier to use but it would be a +# security problem. If you use this script in a VM and only for bundling +# purposes then it shouldn't be a problem. + +# You'll need an internet connection, approximately 1.3Gb of disk space and +# some patience. + +################################################################################ + +set -e # Exit immediately if a command exits with a non-zero status. + +BASE='/home/leap/bitmask.bundle' +mkdir -p $BASE + +# Note: we could use: +# ARCH=`uname -i` +# but it does not work on a VM (tested in i386 returns unknown) +if [[ `getconf LONG_BIT` == "64" ]]; then + ARCH='x86_64-linux-gnu' +else + ARCH='i386-linux-gnu' +fi + +install_dependencies() { + sudo apt-get install -y build-essential python-dev cmake git autoconf \ + libtool liblzo2-dev libqt4-dev libxml2-dev libxslt1-dev qtmobility-dev \ + libsqlite3-dev libffi-dev python-virtualenv +} + +build_boost() { + cd $BASE + + BOOST_NAME='boost_1_56_0' + wget -c http://ufpr.dl.sourceforge.net/project/boost/boost/1.56.0/$BOOST_NAME.tar.bz2 + tar xjf $BOOST_NAME.tar.bz2 + + cd $BOOST_NAME/tools/build/ + ./bootstrap.sh --with-toolset=gcc + ./b2 install --prefix=$BASE/boost + export PATH=$PATH:$BASE/boost/bin/ + + cd $BASE/$BOOST_NAME # boost root + b2 cxxflags=-std=c++0x --with-python --with-filesystem --with-system variant=release link=shared +} + +build_launcher() { + cd $BASE + git clone -b develop https://leap.se/git/bitmask_launcher.git + cd bitmask_launcher + + mkdir build + cd build + + cmake -DBoost_INCLUDE_DIR=$BASE/$BOOST_NAME .. + make +} + +build_openvpn() { + # Build openvpn to support RPATH + cd $BASE + git clone https://github.com/OpenVPN/openvpn.git + cd openvpn + autoreconf -i + LZO_LIBS="/usr/lib/$ARCH/liblzo2.a" ./configure LDFLAGS="-Wl,-rpath,lib/" --disable-snappy --disable-plugin-auth-pam + make -j2 +} + +build_pyside() { + # for more information look at: + # https://github.com/PySide/pyside-setup/blob/master/docs/building/linux.rst + cd $BASE + sudo pip install wheel + + git clone https://github.com/PySide/pyside-setup.git pyside-setup.git + cd pyside-setup.git + python setup.py bdist_wheel --qmake=/usr/bin/qmake-qt4 --version=1.2.2 +} + +set_pyside_environment() { + arch_bits=`getconf LONG_BIT` # '32' or '64' + # from https://github.com/PySide/BuildScripts/blob/master/environment.sh + PYSIDE="$BASE/pyside-setup.git/pyside_install/py2.7-qt4.8.2-${arch_bits}bit-release/" + PYTHONXY='python2.7' + export PATH=$PYSIDE/bin:$PATH + export PYTHONPATH=$PYSIDE/lib/$PYTHONXY/site-packages:$PYSIDE/lib64/$PYTHONXY/site-packages:$PYTHONPATH + export LD_LIBRARY_PATH=$PYSIDE/lib:$LD_LIBRARY_PATH + export PKG_CONFIG_PATH=$PYSIDE/lib/pkgconfig:$PKG_CONFIG_PATH + export DYLD_LIBRARY_PATH=$PYSIDE/lib:$DYLD_LIBRARY_PATH +} + +copy_binaries() { + cd $BASE + ../copy-binaries.sh +} + +create_bundler_paths() { + cd $BASE + cat > bundler.paths << EOF +$BASE/bundler.git/bundler +/usr/lib/python2.7 +/usr/lib/python2.7/plat-linux2 +/usr/lib/python2.7/lib-tk +/usr/lib/python2.7/lib-old +/usr/lib/python2.7/lib-dynload +/usr/local/lib/python2.7/dist-packages +/usr/lib/python2.7/dist-packages +/usr/lib/python2.7/dist-packages/gtk-2.0 +/usr/lib/pymodules/python2.7 +EOF +} + +setup_bundler() { + cd $BASE + + git clone https://github.com/chiiph/bundler.git bundler.git + virtualenv bundler.venv && source bundler.venv/bin/activate + + # install dependencies by hand... + pip install psutil + pip install tuf # used in the launher, it is not in any requirements.txt + + git clone https://github.com/chiiph/protobuf-socket-rpc protobuf-socket-rpc.git + cd protobuf-socket-rpc.git + python setup.py easy_install -Z . + + pip install -r $BASE/bundler.git/pkg/requirements.pip +} + +run_bundler() { + cd $BASE + + # if the virtualenv is not sourced, then source it! + # this is helpful if you want to run this step only + [[ -z "$VIRTUAL_ENV" ]] && source bundler.venv/bin/activate + + set_pyside_environment + + mkdir bundler.output + python bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths --do gitclone pythonsetup $VERSION + python bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths --skip gitclone pythonsetup $VERSION +} + +[[ "$1" == 'nightly' ]] && VERSION='--nightly' + +install_dependencies +build_boost +build_launcher +build_openvpn +build_pyside +copy_binaries +create_bundler_paths +setup_bundler +run_bundler -- cgit v1.2.3