diff options
Diffstat (limited to 'tests/platform-ci/ci-build.sh')
| -rwxr-xr-x | tests/platform-ci/ci-build.sh | 175 |
1 files changed, 137 insertions, 38 deletions
diff --git a/tests/platform-ci/ci-build.sh b/tests/platform-ci/ci-build.sh index be492d32..9bdf75fb 100755 --- a/tests/platform-ci/ci-build.sh +++ b/tests/platform-ci/ci-build.sh @@ -12,9 +12,9 @@ # * AWS credentials as environment variables: # * `AWS_ACCESS_KEY` # * `AWS_SECRET_KEY` -# * ssh private key used to login to remove vm -# * `SSH_PRIVATE_KEY` -# +# * ssh private keys used to clone providers: +# * `bitmask_PROVIDER_SSH_PRIVATE_KEY` +# * `ibex_PROVIDER_SSH_PRIVATE_KEY` # exit if any commands returns non-zero status set -e @@ -39,6 +39,9 @@ ROOTDIR=$(readlink -f "$(dirname $0)") # leap_platform PLATFORMDIR=$(readlink -f "${ROOTDIR}/../..") +# deb repo component to configure +COMPONENT=${COMPONENT:-"master"} + # In the gitlab CI pipeline leap is installed in a different # stage by bundle. To debug you can run a single CI job locally # so we install leap_cli as gem here. @@ -63,22 +66,50 @@ deploy() { LEAP_CMD deploy "$TAG" } +leap_info() { + echo "Running leap info on $TAG" + LEAP_CMD info "${TAG}" +} + test() { LEAP_CMD test "$TAG" } +ssh_setup() { + # set the provider name from the first argument passed to the function + provider_name=$1 + # set CI_SSH_SECRET_PRIVATE_KEY to the variable name keyed off of the provider_name + CI_SSH_SECRET_PRIVATE_KEY=${provider_name}_PROVIDER_SSH_PRIVATE_KEY + # Set the SSH_PRIVATE_KEY to the value provided in the CI runner secret variable setting in gitlab + SSH_PRIVATE_KEY=${!CI_SSH_SECRET_PRIVATE_KEY} + echo "Working with provider: $provider_name" + [ -z "$SSH_PRIVATE_KEY" ] && fail "${provider_name}_PROVIDER_SSH_PRIVATE_KEY is not set - please provide it as env variable." + # Configure ssh keypair + [ -d ~/.ssh ] || /bin/mkdir ~/.ssh + /bin/echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa + /bin/chmod 600 ~/.ssh/id_rsa + /bin/cp "${ROOTDIR}/provider/users/gitlab-runner-${provider_name}/gitlab-runner-${provider_name}_ssh.pub" ~/.ssh/id_rsa.pub +} + build_from_scratch() { - # when using gitlab-runner locally, CI_BUILD_ID is always 1 which + # setup ssh keys + ssh_setup platform + + # allow passing into the function the services, use a default set if empty + SERVICES=$1 + if [ -z "$SERVICES" ] + then + SERVICES='couchdb,soledad,mx,webapp,tor_relay,monitor' + fi + + # when using gitlab-runner locally, CI_JOB_ID is always 1 which # will conflict with running/terminating AWS instances in subsequent runs # therefore we pick a random number in this case [ "${CI_JOB_ID}" == "1" ] && CI_JOB_ID="000${RANDOM}" # create node(s) with unique id so we can run tests in parallel NAME="citest${CI_JOB_ID:-0}" - - TAG='single' - SERVICES='couchdb,soledad,mx,webapp,tor,monitor' # leap_platform/tests/platform-ci/provider PROVIDERDIR="${ROOTDIR}/provider" @@ -88,12 +119,11 @@ build_from_scratch() { # Create cloud.json needed for `leap vm` commands using AWS credentials which jq || ( apt-get update -y && apt-get install jq -y ) - # Dsiable xtrace + # Disable xtrace set +x [ -z "$AWS_ACCESS_KEY" ] && fail "\$AWS_ACCESS_KEY is not set - please provide it as env variable." [ -z "$AWS_SECRET_KEY" ] && fail "\$AWS_SECRET_KEY is not set - please provide it as env variable." - [ -z "$SSH_PRIVATE_KEY" ] && fail "\$SSH_PRIVATE_KEY is not set - please provide it as env variable." /usr/bin/jq ".platform_ci.auth |= .+ {\"aws_access_key_id\":\"$AWS_ACCESS_KEY\", \"aws_secret_access_key\":\"$AWS_SECRET_KEY\"}" < cloud.json.template > cloud.json # Enable xtrace again only if it was set at beginning of script @@ -102,7 +132,8 @@ build_from_scratch() { [ -d "./tags" ] || mkdir "./tags" /bin/echo "{\"environment\": \"$TAG\"}" | /usr/bin/json_pp > "${PROVIDERDIR}/tags/${TAG}.json" - pwd + # configure deb repo component + echo '{}' | jq ".sources.platform.apt |= { \"source\": \"http://deb.leap.se/platform\", \"component\": \"${COMPONENT}\" }" > common.json # remove old cached nodes echo "Removing old cached nodes..." @@ -123,53 +154,126 @@ build_from_scratch() { echo "Running leap node init on TAG: $TAG" LEAP_CMD node init "$TAG" - echo "Running leap info on $TAG" - LEAP_CMD info "${TAG}" } run() { - echo "Cloning $1 repo: $2" - git clone -q --depth 1 "$2" - cd "$1" - git rev-parse HEAD - echo -n "Operating in the $1 directory: " - pwd - echo "Listing current node information..." - LEAP_CMD list - echo "Attempting a deploy..." - deploy - echo "Attempting to run tests..." - test + provider_name=$1 + provider_URI=$2 + platform_branch=$3 + + # setup ssh keys + ssh_setup "$provider_name" + + # Setup the provider repository + echo "Setting up the provider repository: $provider_name by cloning $provider_URI" + git clone -q --depth 1 "$provider_URI" + cd "$provider_name" + echo -n "$provider_name repo at revision: " + git rev-parse HEAD + echo -n "Operating in the $provider_name directory: " + pwd + + + # If the third argument is set make sure we are on that platform branch + if [[ -n $platform_branch ]] + then + echo "Checking out $platform_branch branch of platform" + cd "$PLATFORMDIR" + git checkout -B "$platform_branch" + fi + + cd "${ROOTDIR}/${provider_name}" + echo "Listing current node information..." + LEAP_CMD list + + # Do the deployment + echo "Attempting a deploy..." + LEAP_CMD cert renew "$CI_JOB_NAME" + deploy + echo "Attempting to run tests..." + test +} + +soledad_migration() { + # check the version of soledad installed + # if the version is not greater than 0.9, we need to do the migration + if ! LEAP_CMD run "dpkg --compare-versions \$(dpkg -l |grep soledad-server|grep ^ii|awk '{ print \$3}') gt 0.8" vm |grep -q oops + then + echo "Need to migrate from soledad 0.8!" + if ! LEAP_CMD run 'systemctl stop leap-mx' vm + then fail + fi + if ! LEAP_CMD run 'systemctl stop soledad-server' vm + then fail + fi + if ! LEAP_CMD run --stream '/usr/share/soledad-server/migration/0.9/migrate.py --log-file /dev/stdout --verbose --do-migrate | tee /var/log/leap/soledad_migration.log' vm + then fail + fi + if ! LEAP_CMD run 'systemctl start leap-mx' vm + then fail + fi + if ! LEAP_CMD run 'systemctl start soledad-server' vm + then fail + fi + fi } upgrade_test() { - # Checkout stable branch containing last release + # Checkout stable branch containing previous stable release # and deploy this cd "$PLATFORMDIR" - git remote add leap https://leap.se/git/leap_platform + # due to cache, this remote is sometimes already added + git remote add leap https://leap.se/git/leap_platform || true git fetch leap - git checkout -b leap_stable remotes/leap/stable + echo "Checking out leap/stable" + git checkout -b leap_stable remotes/leap/stable || true + echo -n "Current version: " + git rev-parse HEAD # After checking out a different platform branch # bundle install is needed again cd "$ROOTDIR" /usr/local/bin/bundle install cd "$PROVIDERDIR" - - build_from_scratch + LEAP_CMD --version + build_from_scratch 'couchdb,soledad,mx,webapp,tor,monitor' deploy + leap_info + # In 0.9 leap info did not output apt sources, so we do it manually + # but can remove it for next release + cat /etc/apt/sources.list.d/* test # Checkout HEAD of current branch and re-deploy cd "$PLATFORMDIR" - git checkout "$CI_COMMIT_REF" + echo "Checking out: $CI_COMMIT_SHA" + git checkout "$CI_COMMIT_SHA" + echo -n "Current version: " + git rev-parse HEAD # After checking out a different platform branch # bundle install is needed again cd "$ROOTDIR" /usr/local/bin/bundle install cd "$PROVIDERDIR" + LEAP_CMD --version + + # due to the 'tor' service no longer being valid in 0.10, we need to change + # that service to 'tor_relay'. This is done by changing the services array + # with jq to be set to the full correct list of services + jq '.services = ["couchdb","soledad","mx","webapp","tor_relay","monitor"]' < nodes/${NAME}.json deploy + + # pre-migration test + # allowed to fail because when a migration is needed, soledad-server refuses to start + test || /bin/true + + # check for soledad migration, and run it if necessary + soledad_migration + + leap_info + + # run the test again, this should succeed test cleanup @@ -192,12 +296,6 @@ cleanup() { # Ensure we don't output secret stuff to console even when running in verbose mode with -x set +x -# Configure ssh keypair -[ -d ~/.ssh ] || /bin/mkdir ~/.ssh -/bin/echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa -/bin/chmod 600 ~/.ssh/id_rsa -/bin/cp "${ROOTDIR}/provider/users/gitlab-runner/gitlab-runner_ssh.pub" ~/.ssh/id_rsa.pub - # Enable xtrace again only if it was set at beginning of script [[ $xtrace == true ]] && set -x @@ -208,15 +306,16 @@ case "$CI_JOB_NAME" in ;; mail.bitmask.net) TAG='demomail' - run bitmask ssh://gitolite@leap.se/bitmask + run bitmask ssh://gitolite@leap.se/bitmask master ;; demo.bitmask.net) TAG='demovpn' - run bitmask ssh://gitolite@leap.se/bitmask + run bitmask ssh://gitolite@leap.se/bitmask master ;; deploy_test*) build_from_scratch deploy + leap_info test cleanup ;; |