From c71ffaeff607a7af02db997d86bde72c98516374 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Fri, 24 Oct 2014 16:17:47 -0300 Subject: Add REUSE_BINARIES capability. Using this new feature we can reduce the bundling time significatively since we don't need to spend much time compiling stuff that shouldn't change. Move dependencies to the requirements file. Use helper variable to shorten the bundler command call. --- createbundle.sh | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) (limited to 'createbundle.sh') diff --git a/createbundle.sh b/createbundle.sh index 825fb43..e8b7583 100755 --- a/createbundle.sh +++ b/createbundle.sh @@ -52,6 +52,7 @@ else fi install_dependencies() { + sudo apt-get update # update apt, otherwise some packages may not be found sudo apt-get install -y build-essential python-dev cmake git autoconf \ libtool liblzo2-dev libqt4-dev libxml2-dev libxslt1-dev qtmobility-dev \ libsqlite3-dev libffi-dev python-virtualenv @@ -145,10 +146,6 @@ setup_bundler() { git clone https://github.com/leapcode/bitmask_bundler.git bitmask_bundler.git virtualenv bundler.venv && source bundler.venv/bin/activate - # install dependencies by hand... - pip install psutil - pip install tuf # used in the launher, it is not in any requirements.txt - git clone https://github.com/chiiph/protobuf-socket-rpc protobuf-socket-rpc.git cd protobuf-socket-rpc.git python setup.py easy_install -Z . @@ -156,28 +153,50 @@ setup_bundler() { pip install -r $BASE/bitmask_bundler.git/pkg/requirements.pip } +clean() { + cd $BASE + rm -fr bitmask_bundler.git bundler.venv protobuf-socket-rpc.git bundler.output +} + run_bundler() { cd $BASE # if the virtualenv is not sourced, then source it! - # this is helpful if you want to run this step only + # this is helpful if you want to run only this step [[ -z "$VIRTUAL_ENV" ]] && source bundler.venv/bin/activate set_pyside_environment mkdir bundler.output - python bitmask_bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths --do gitclone pythonsetup $VERSION - python bitmask_bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths --skip gitclone pythonsetup $VERSION + + # Use a shortcut for the bundler command + bundler="python bitmask_bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths" + + $bundler --do gitclone + $bundler --do gitcheckout --versions-file $BASE/bitmask.json + + $bundler --do pythonsetup + $bundler --skip gitclone gitcheckout pythonsetup --versions-file $BASE/bitmask.json } [[ "$1" == 'nightly' ]] && VERSION='--nightly' -install_dependencies -build_boost -build_launcher -build_openvpn -build_pyside -copy_binaries -create_bundler_paths +REUSE_BINARIES=$BASE/reuse-binaries.lock + +if [[ ! -f $REUSE_BINARIES ]]; then + install_dependencies + build_boost + build_launcher + build_openvpn + build_pyside + copy_binaries + create_bundler_paths +else + echo "Reusing existing binaries, cleaning up before creating a new bundle..." + clean +fi + setup_bundler run_bundler + +echo 'If you remove this file the createbundle.sh script will rebuild all the binaries.' > $REUSE_BINARIES -- cgit v1.2.3 From 7d943532210834ec2e839dcdd257b3f9c4823e13 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Fri, 24 Oct 2014 16:20:31 -0300 Subject: Use json file to define versions to be bundled. Also improve logging support to print each component's name. --- createbundle.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'createbundle.sh') diff --git a/createbundle.sh b/createbundle.sh index e8b7583..90bf53d 100755 --- a/createbundle.sh +++ b/createbundle.sh @@ -172,15 +172,13 @@ run_bundler() { # Use a shortcut for the bundler command bundler="python bitmask_bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths" - $bundler --do gitclone - $bundler --do gitcheckout --versions-file $BASE/bitmask.json + $bundler --do gitclone --versions-file $BASE/../bitmask.json + $bundler --do gitcheckout --versions-file $BASE/../bitmask.json - $bundler --do pythonsetup - $bundler --skip gitclone gitcheckout pythonsetup --versions-file $BASE/bitmask.json + $bundler --do pythonsetup --versions-file $BASE/../bitmask.json + $bundler --skip gitclone gitcheckout pythonsetup --versions-file $BASE/../bitmask.json } -[[ "$1" == 'nightly' ]] && VERSION='--nightly' - REUSE_BINARIES=$BASE/reuse-binaries.lock if [[ ! -f $REUSE_BINARIES ]]; then -- cgit v1.2.3 From bb67f3f2761451dceefc5648f750d61b01fe8daa Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Mon, 27 Oct 2014 17:39:01 -0300 Subject: Install manually troublesome deps. --- createbundle.sh | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'createbundle.sh') diff --git a/createbundle.sh b/createbundle.sh index 90bf53d..4d4dfbe 100755 --- a/createbundle.sh +++ b/createbundle.sh @@ -146,6 +146,10 @@ setup_bundler() { git clone https://github.com/leapcode/bitmask_bundler.git bitmask_bundler.git virtualenv bundler.venv && source bundler.venv/bin/activate + # HACK: install these dependencies manually since they can't be installed normally due not passing some pip checks. + pip install --upgrade pip + pip install --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec u1db dirspec + git clone https://github.com/chiiph/protobuf-socket-rpc protobuf-socket-rpc.git cd protobuf-socket-rpc.git python setup.py easy_install -Z . -- cgit v1.2.3 From a9b4e2bbf9f6c8712d2d814d01165ad06481d4cc Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Mon, 27 Oct 2014 17:58:50 -0300 Subject: Add checks for mandatory files. Also factor out bundler arguments. --- createbundle.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'createbundle.sh') diff --git a/createbundle.sh b/createbundle.sh index 4d4dfbe..1336263 100755 --- a/createbundle.sh +++ b/createbundle.sh @@ -40,6 +40,11 @@ set -e # Exit immediately if a command exits with a non-zero status. BASE='/home/leap/bitmask.bundle' + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +VERSIONS_FILE="$SCRIPT_DIR/bitmask.json" +BINARY_COPIER="$SCRIPT_DIR/copy-binaries.sh" + mkdir -p $BASE # Note: we could use: @@ -121,7 +126,7 @@ set_pyside_environment() { copy_binaries() { cd $BASE - ../copy-binaries.sh + $BINARY_COPIER } create_bundler_paths() { @@ -174,17 +179,27 @@ run_bundler() { mkdir bundler.output # Use a shortcut for the bundler command - bundler="python bitmask_bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths" + bundler="python bitmask_bundler.git/bundler/main.py --workon bundler.output --binaries binaries/ --paths-file bundler.paths --versions-file $VERSIONS_FILE" - $bundler --do gitclone --versions-file $BASE/../bitmask.json - $bundler --do gitcheckout --versions-file $BASE/../bitmask.json + $bundler --do gitclone + $bundler --do gitcheckout - $bundler --do pythonsetup --versions-file $BASE/../bitmask.json - $bundler --skip gitclone gitcheckout pythonsetup --versions-file $BASE/../bitmask.json + $bundler --do pythonsetup + $bundler --skip gitclone gitcheckout pythonsetup } REUSE_BINARIES=$BASE/reuse-binaries.lock +if [[ ! -f $BINARY_COPIER ]]; then + echo "ERROR: missing $BINARY_COPIER file." + exit 1 +fi + +if [[ ! -f $VERSIONS_FILE ]]; then + echo "ERROR: missing $VERSIONS_FILE file." + exit 1 +fi + if [[ ! -f $REUSE_BINARIES ]]; then install_dependencies build_boost -- cgit v1.2.3