Enhance ci-build.sh for latest CI builds.

 . Reorganize script to allow for multiple builds
 . Add latest build, pulling from the ibex provider
 . Run the build as the cirunner unprivileged user
 . Set pipefail because job is run within a pipe
 . Change name of 'build' stage to 'deploy'
 . Setup an environment for the latest CI deployment

2 files changed, 97 insertions, 53 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ab2d5aa5..8d3afaa5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-image: leapcode/ruby
+image: 0xacab.org:4567/leap/gitlab-buildpackage:ruby
 
 # This is for caching the gems not only between the stages, but also persistent
 # on the gitlab-runner so we don't need to install from scratch on every pipeline
@@ -14,7 +14,7 @@ before_script:
 stages:
   - setup
   - syntax
-  - build
+  - deploy
 
 setup:
   stage: setup
@@ -24,27 +24,27 @@ setup:
 lint:
   stage: syntax
   script:
-    - /usr/local/bin/bundle exec rake lint
+    - su -c '/usr/local/bin/bundle exec rake lint' cirunner
 
 syntax:
   stage: syntax
   script:
-    - /usr/local/bin/bundle exec rake syntax
+    - su -c '/usr/local/bin/bundle exec rake syntax' cirunner
 
 validate:
   stage: syntax
   script:
-    - /usr/local/bin/bundle exec rake validate
+    - su -c '/usr/local/bin/bundle exec rake validate' cirunner
 
 templates:
   stage: syntax
   script:
-    - /usr/local/bin/bundle exec rake templates
+    - su -c '/usr/local/bin/bundle exec rake templates' cirunner
 
 catalog:
   stage: syntax
   script:
-    - /usr/local/bin/bundle exec rake catalog
+    - su -c '/usr/local/bin/bundle exec rake catalog' cirunner
 
 #rspec:
 #  stage: rspec
@@ -52,6 +52,20 @@ catalog:
 #    - /usr/local/bin/bundle exec rake spec
 
 build:
-  stage: build
+  stage: deploy
   script:
-    - /usr/bin/unbuffer ./ci-build.sh | /usr/bin/ts -s
+    - su -c '/usr/bin/unbuffer ./ci-build.sh | /usr/bin/ts -s' cirunner
+
+# Latest job will only run on the master branch, which means all merge requests
+# that are created from branches don't get to deploy to the latest-ci server.
+# When a merge request is merged, then the latest job will deploy the code to
+# the latest provider, and the deployment will be recorded in an environment
+# named 'latest'
+latest:
+  stage: deploy
+  environment:
+    name: latest
+  only:
+  - master
+  script:
+    - su -c '/usr/bin/unbuffer ./ci-build.sh | /usr/bin/ts -s' cirunner
diff --git a/tests/platform-ci/ci-build.sh b/tests/platform-ci/ci-build.sh
index af1dba0f..869e7517 100755
--- a/tests/platform-ci/ci-build.sh
+++ b/tests/platform-ci/ci-build.sh
@@ -15,75 +15,105 @@
 #   * ssh private key used to login to remove vm
 #     * `SSH_PRIVATE_KEY`
 #
-# Todo:
-#   - Running locally works fine, now use it in gitlab CI ( which ssh-key ? create cloud.json from env vars )
-#   - Speed up vm boot if possible ( right now 3-4mins )
 
 # exit if any commands returns non-zero status
 set -e
+# because the ci-build is running in a pipe we need to also set the following
+# so exit codes will be caught correctly.
+set -o pipefail
 
 # leap_platform/tests/platform-ci
 # shellcheck disable=SC2086
 ROOTDIR=$(readlink -f "$(dirname $0)")
 
-# leap_platform/tests/platform-ci/provider
-PROVIDERDIR="${ROOTDIR}/provider"
-
 # leap_platform
 PLATFORMDIR=$(readlink -f "${ROOTDIR}/../..")
 
-LEAP_CMD="/usr/local/bin/bundle exec leap -v2 --yes"
+LEAP_CMD() {
+  /usr/local/bin/bundle exec leap -v2 --yes "$@"
+}
+
+deploy() {
+  LEAP_CMD deploy "$TAG"
+}
+
+test() {
+  LEAP_CMD test "$TAG"
+}
+
+build_from_scratch() {
+  # leap_platform/tests/platform-ci/provider
+  PROVIDERDIR="${ROOTDIR}/provider"
+  /bin/echo "Provider directory: ${PROVIDERDIR}"
+  cd "$PROVIDERDIR"
 
-# create node(s) with unique id so we can run tests in parallel
-NAME="citest${CI_BUILD_ID}"
-# when using gitlab-runner locally, CI_BUILD_ID is always 1 which
-# will conflict with running/terminating AWS instances in subsequent runs
-# therefore we pick a random number in this case
-[ "$CI_BUILD_ID" -eq "1" ] && NAME+="000${RANDOM}"
+  # Create cloud.json needed for `leap vm` commands using AWS credentials
+  which jq || ( apt-get update -y && apt-get install jq -y )
+  /usr/bin/jq ".platform_ci.auth |= .+ {\"aws_access_key_id\":\"$AWS_ACCESS_KEY\", \"aws_secret_access_key\":\"$AWS_SECRET_KEY\"}" < cloud.json.template > cloud.json
 
-TAG='single'
-SERVICES='couchdb,soledad,mx,webapp,tor,monitor'
-SEEDS=''
+  [ -d "./tags" ] || mkdir "./tags"
+  /bin/echo "{\"environment\": \"$TAG\"}" | /usr/bin/json_pp > "${PROVIDERDIR}/tags/${TAG}.json"
+
+  pwd
+  LEAP_CMD vm status "$TAG"
+  # shellcheck disable=SC2086
+  LEAP_CMD vm add "$NAME" services:"$SERVICES" tags:"$TAG" $SEEDS
+  LEAP_CMD compile "$TAG"
+  LEAP_CMD vm status "$TAG"
+
+  LEAP_CMD node init "$TAG"
+  LEAP_CMD info "${TAG}"
+}
 
 #
 # Main
 #
 
-
 /bin/echo "CI directory: ${ROOTDIR}"
-/bin/echo "Provider directory: ${PROVIDERDIR}"
 /bin/echo "Platform directory: ${PLATFORMDIR}"
-cd "$PROVIDERDIR"
 
 # Ensure we don't output secret stuff to console even when running in verbose mode with -x
 set +x
 
-# Create cloud.json needed for `leap vm` commands using AWS credentials
-which jq || ( apt-get update -y && apt-get install jq -y )
-/usr/bin/jq ".platform_ci.auth |= .+ {\"aws_access_key_id\":\"$AWS_ACCESS_KEY\", \"aws_secret_access_key\":\"$AWS_SECRET_KEY\"}" < cloud.json.template > cloud.json
-
 # Configure ssh keypair
 [ -d ~/.ssh ] || /bin/mkdir ~/.ssh
 /bin/echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
 /bin/chmod 600 ~/.ssh/id_rsa
-/bin/cp users/gitlab-runner/gitlab-runner_ssh.pub ~/.ssh/id_rsa.pub
-
-[ -d "./tags" ] || mkdir "./tags"
-/bin/echo "{\"environment\": \"$TAG\"}" | /usr/bin/json_pp > "${PROVIDERDIR}/tags/${TAG}.json"
-
-$LEAP_CMD vm status "$TAG"
-# shellcheck disable=SC2086
-$LEAP_CMD vm add "$NAME" services:"$SERVICES" tags:"$TAG" $SEEDS
-$LEAP_CMD compile "$TAG"
-$LEAP_CMD vm status "$TAG"
-
-$LEAP_CMD node init "$TAG"
-
-# Deploy and test
-$LEAP_CMD deploy "$TAG"
-$LEAP_CMD info "${TAG}"
-$LEAP_CMD test "$TAG"
-
-# if everything succeeds, destroy the vm
-$LEAP_CMD vm rm "${TAG}"
-[ -f "nodes/${NAME}.json" ] && /bin/rm "nodes/${NAME}.json"
+/bin/cp "${ROOTDIR}/provider/users/gitlab-runner/gitlab-runner_ssh.pub" ~/.ssh/id_rsa.pub
+
+case "$CI_BUILD_STAGE" in
+  build)
+    # create node(s) with unique id so we can run tests in parallel
+    NAME="citest${CI_BUILD_ID}"
+    # when using gitlab-runner locally, CI_BUILD_ID is always 1 which
+    # will conflict with running/terminating AWS instances in subsequent runs
+    # therefore we pick a random number in this case
+    [ "$CI_BUILD_ID" -eq "1" ] && NAME+="000${RANDOM}"
+
+    TAG='single'
+    SERVICES='couchdb,soledad,mx,webapp,tor,monitor'
+    SEEDS=''
+    build_from_scratch
+    # Deploy and test
+    deploy
+    test
+    # if everything succeeds, destroy the vm
+    LEAP_CMD vm rm "${TAG}"
+    [ -f "nodes/${NAME}.json" ] && /bin/rm "nodes/${NAME}.json"
+    ;;
+  latest)
+    TAG='latest'
+    echo "Cloning ibex provider..."
+    git clone -q --depth 1 ssh://gitolite@leap.se/ibex
+    cd ibex
+    git rev-parse HEAD
+    echo -n "Operating in the ibex directory: "
+    pwd
+    echo "Listing current node information..."
+    LEAP_CMD list
+    echo "Attempting a deploy..."
+    deploy
+    echo "Attempting to run tests..."
+    test
+    ;;
+esac