path: root/master.cfg

# -*- python -*-
# ex: set syntax=python:

import textwrap
import ConfigParser

config = ConfigParser.ConfigParser()
config.read('conf.cfg')

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### BUILDSLAVES

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
import slaves
reload(slaves)
c['slaves'] = slaves.slaves

# 'protocols' contains information about protocols which master will use for
# communicating with slaves.
# You must define at least 'port' option that slaves could connect to your master
# with this protocol.
# 'port' must match the value configured into the buildslaves (with their
# --master option)
PORT_WEB = 8010           # Buildbot webserver port
PORT_MASTER = 9989      # Port where buildbot master listen buildworkers

c['protocols'] = {'pb': {'port': PORT_MASTER}}

####### CHANGESOURCES
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.

github_repos_username = 'leapcode'
default_branch = 'develop'
order_repos_index = 2
REPOS=[
    ('leap_pycommon', default_branch, 1, 'leap.common'),
    ('soledad', default_branch, 2, 'leap.soledad'),
    ('keymanager', default_branch, 3, 'leap.keymanager'),
    ('leap_mail', default_branch, 4, 'leap.mail'),
    ('bitmask_client', default_branch, 5, 'leap.bitmask'),
]

def github_repo_url(repo_name):
    return 'https://github.com/' + github_repos_username + '/' + repo_name + '.git'


all_repositories = {
    r'https://github.com/leapcode/leap_pycommon.git': 'leap_pycommon',
    r'https://github.com/leapcode/soledad.git': 'soledad',
    r'https://github.com/leapcode/keymanager.git': 'keymanager',
    r'https://github.com/leapcode/leap_mail.git': 'leap_mail',
    r'https://github.com/leapcode/bitmask_client.git': 'bitmask_client',
}
def codebaseGenerator(chdict):
    return all_repositories[chdict['repository']]

c['codebaseGenerator'] = codebaseGenerator

from buildbot.plugins import changes
poll_interval_minutes = 5
c['change_source'] = [changes.GitPoller(repourl=github_repo_url(repo_name),
                                        branches=[repo_branch],
                                        workdir=repo_name,
                                        pollInterval=poll_interval_minutes * 60)
                      for repo_name, repo_branch, _, _ in REPOS]

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.

from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler, CodebaseParameter
from buildbot.changes.filter import ChangeFilter

from buildbot.plugins import schedulers
c['schedulers'] = []

def all_codebases(repo_name):
    codebases = {}

    repo_index = [repo[order_repos_index] for repo in REPOS if repo[0] is repo_name][0]
    for repo_name, repo_branch, _, namespace, in sorted(REPOS, key = lambda repo: repo[order_repos_index])[0:repo_index]:
        codebases[repo_name] = {
            'repository' : github_repo_url(repo_name),
            'branch' : None,
            'revision' : None
        }

    return codebases

def all_codebases_names(repo_name):
    codebases = []

    repo_index = [repo[order_repos_index] for repo in REPOS if repo[0] is repo_name][0]
    for codebase_repo_name, codebase_repo_branch, _, _, in sorted(REPOS, key = lambda repo: repo[order_repos_index])[0:repo_index]:
        codebases.append(CodebaseParameter(codebase=codebase_repo_name, branch=codebase_repo_branch, repository=github_repo_url(codebase_repo_name)))

    return codebases

for repo_name, repo_branch, order_, _ in REPOS:
    c['schedulers'].append(AnyBranchScheduler(
        name=repo_name,
        change_filter = ChangeFilter(codebase=repo_name),
        codebases = all_codebases(repo_name),
        treeStableTimer = 60,
        builderNames=['builder_' + repo_name]))

    c['schedulers'].append(ForceScheduler(
        name="force_build_of_" + repo_name,
        codebases = all_codebases_names(repo_name),
        builderNames=['builder_' + repo_name]))

c['schedulers'].append(ForceScheduler(
    name="force_build_of_wheels",
    builderNames=['builder_wheels']))

c['schedulers'].append(ForceScheduler(
    name="force_build_of_bundler",
    builderNames=['builder_bundler']))

c['schedulers'].append(
    schedulers.Nightly(name='nightly_bundle',
                       branch=None,
                       builderNames=['builder_bundler'],
                       hour=0, minute=0))
c['schedulers'].append(
    schedulers.Nightly(name='nightly_bitmask_client',
                       branch=None,
                       builderNames=['builder_bitmask_client'],
                       hour=23, minute=55))

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

from buildbot.process.factory import BuildFactory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.config import BuilderConfig

def add_repo_to_factory(factory, repo_name, git_branch, namespace, venv_name):
    install_requirements = 'pkg/pip_install_requirements.sh --use-leap-wheels'
    install_requirements_tests = "if [ -f pkg/requirements-testing.pip ]; then pkg/pip_install_requirements.sh --testing --use-leap-wheels; fi"
    install = "python setup.py develop"

    workdir = repo_name
    sandbox_path = {'PATH':  "../" + venv_name + '/bin/' + ':${PATH}'}
    sandbox_path_soledad = {'PATH':  "../../" + venv_name + '/bin/' + ':${PATH}'}
    repo_url = github_repo_url(repo_name)

    factory.addStep(
        Git(repourl=repo_url, branch=git_branch, workdir=workdir, codebase=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url))

    if 'bitmask_client' in repo_name:
        factory.addSteps([
            ShellCommand(command='pkg/postmkvenv.sh',env=sandbox_path,haltOnFailure=False, workdir=workdir, name="postmkenv"),
            ShellCommand(command='make',env=sandbox_path,haltOnFailure=False, workdir=workdir, name="make")
        ])
    if 'soledad.git' in repo_url:
        for subpackage in ["common", "client", "server"]:
            if 'keymanager' in venv_name and subpackage is not "server" or 'keymanager' is not venv_name: # keymanager doesn't need soledad.server
                factory.addSteps([
                    ShellCommand(command=install_requirements, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="reqs: " + repo_name+"."+subpackage),
                    ShellCommand(command=install_requirements_tests, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="test reqs: " + repo_name+"."+subpackage),
                    ShellCommand(command=install, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="Install " + repo_name+"."+subpackage)
                ])
    else:
        factory.addSteps([
            ShellCommand(command=install_requirements, env=sandbox_path, haltOnFailure=False, workdir=workdir, name="reqs: " + repo_name),
            ShellCommand(command=install_requirements_tests, env=sandbox_path, haltOnFailure=False, workdir=workdir, name="test reqs: " + repo_name),
            ShellCommand(command=install, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="Install " + repo_name)
        ])

def create_builder(repo_name):
    builder_name = 'builder_' + repo_name
    venv_name = "virtualenv_ci_" + builder_name
    venv_path = {'PATH':  "./" + venv_name + '/bin' + ':${PATH}'}
    venv_path_factory = {'PATH':  "../" + venv_name + '/bin' + ':${PATH}'}

    factory = BuildFactory()
    factory.addSteps([
        ShellCommand(command=["rm", "-rf", venv_name], haltOnFailure=True, workdir=".", name="Remove previous virtualenv"),
        ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv"),
        ShellCommand(command=['pip', 'install', '-U', 'pip', 'setuptools', 'coverage'], env=venv_path, workdir=".", name="Update setuptools")
    ])

    repo_index = [repo[order_repos_index] for repo in REPOS if repo[0] is repo_name][0]
    for repo_name, repo_branch, _, namespace, in sorted(REPOS, key = lambda repo: repo[order_repos_index])[0:repo_index]:
        add_repo_to_factory(factory, repo_name, repo_branch, namespace, venv_name)

    factory.addSteps([
        ShellCommand(command=['pep8', '.'],env=venv_path_factory,haltOnFailure=False, workdir=repo_name, name="pep8 on " + repo_name)])

    if namespace is not '':
        if repo_name is 'bitmask_client':
            factory.addStep(
                ShellCommand(command=['xvfb-run', 'coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace))
        else:
            factory.addStep(
                ShellCommand(command=['coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace))

        factory.addSteps([
            ShellCommand(command=['coverage', 'html'], env=venv_path, workdir='.', name="generate html coverage report for " +namespace),
            ShellCommand(command=publish_coverage_reports_command('htmlcov', repo_name), workdir='.', doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))))
        ])

    publish_leap_wheels(factory, repo_name, venv_path_factory, doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))))

    if repo_name == 'bitmask_client':
        publish_sumo = publish_sumo_command('`ls -t *SUMO.tar.gz | head -1`')

        factory.addSteps([
            ShellCommand(command=['make', 'sumo_tarball_latest'],
                          env=venv_path_factory, workdir=repo_name,
                          doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))),
                          name="make sumo tarball"),
             ShellCommand(command=publish_sumo,
                          env=venv_path_factory, workdir=repo_name + "/dist",
                          doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))),
                          name="publish sumo to ftp")
            ])


    return BuilderConfig(name=builder_name, slavenames=slaves.names(), factory=factory)

def publish_coverage_reports_command(location, repo_name):
    target_directory = config.get('ftp', 'coverage_reports_target_directory') + '/' + repo_name + '_' + '`git -C ' + repo_name + ' describe`'

    return ftp_publish_dir_command(location, target_directory)

def publish_leap_wheels(factory, repo_name, env, doStepIf):
    env_soledad = {'PATH':  env['PATH'].replace('../', '../../', 1)}

    if repo_name == 'soledad':
        for subpackage in ["common", "client", "server"]:
            factory.addSteps([
                ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage, name="leap wheels for " + repo_name+"."+subpackage),
                ShellCommand(command=publish_leap_wheels_soledad(subpackage, '`ls -t *.whl | head -1`'), env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage+'/dist', name="publish leap wheels for " + repo_name+"."+subpackage)])
    else:
        factory.addSteps([
            ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env, doStepIf=doStepIf, workdir=repo_name, name="Generate leap wheels for "+repo_name),
            ShellCommand(command=publish_leap_wheels_command(repo_name, '`ls -t *.whl | head -1`'), env=env, doStepIf=doStepIf, workdir=repo_name + '/dist', name="Publish leap wheels for "+repo_name)
        ])

def publish_leap_wheels_command(repo_name, location):
    directory = config.get('ftp', 'leap_wheels_directory')
    command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.' + repo_name + '-latest.whl')

    return command

def publish_leap_wheels_soledad(subpackage, location):
    directory = config.get('ftp', 'leap_wheels_directory')
    command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.soledad.' + subpackage + '-latest.whl')

    return command

def publish_sumo_command(location):
    directory = config.get('ftp', 'sumo_target_directory')
    command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.bitmask-latest-SUMO.tar.gz')

    return command

def ftp_soft_link(filename, target_directory, symlink_name):
    return ftp_ssh_command('ln -sf ' + target_directory + '/' + filename + ' ' + target_directory + '/' + symlink_name)

def ftp_ssh_command(command):
    ssh_port = config.get('ftp', 'ssh_port')
    ssh_key = config.get('ftp', 'ssh_key')
    user = config.get('ftp', 'user')
    server = config.get('ftp', 'server')

    ssh_command = ['ssh',
                   "-i", ssh_key,
                   '-p', ssh_port,
                   user + '@' + server,
                   '"' + command + '"']

    # Flatten to a string so that a shell executes de command, and
    # expands ~
    return ' '.join(ssh_command)

def ftp_publish_dir_command(from_dir, to_dir):
    return ftp_publish_command(from_dir + "/*", to_dir)

def ftp_publish_command(from_location, to_location):
    ssh_port = config.get('ftp', 'ssh_port')
    ssh_key = config.get('ftp', 'ssh_key')
    user = config.get('ftp', 'user')
    server = config.get('ftp', 'server')

    ssh_mkdir_command = ['ssh',
                         "-i", ssh_key,
                         '-p', ssh_port,
                         user + '@' + server,
                         '"mkdir ' + to_location + '"']

    scp_command = ['scp',
                   '-i', ssh_key,
                   '-P', ssh_port,
                   '-r', from_location,
                   '"' + user + '@' + server + ':' + to_location +'"']
    ssh_command = ['ssh',
                   "-i", ssh_key,
                   '-p', ssh_port,
                   user + '@' + server,
                   '"chmod -R g+r ' + to_location + ' && chown -R ' + user + ':www-data ' + to_location + '"']
    # Flatten to a string so that a shell executes de command, and
    # expands ~
    return ' '.join(ssh_mkdir_command) + ' ; ' + ' '.join(scp_command) + ' && ' + ' '.join(ssh_command)

def make_wheel_builder():
    builder_name = "builder_wheels"
    venv_name = "virtualenv_wheels"
    factory = BuildFactory()

    generate_wheels = 'pkg/generate_wheels.sh'
    publish_wheels = publish_wheels_command()

    sandbox_path_top = {'PATH':  "./" + venv_name + '/bin' + ':${PATH}'}
    sandbox_path = {'PATH':  "../" + venv_name + '/bin' + ':${PATH}'}
    sandbox_path_soledad = {'PATH':  "../../" + venv_name + '/bin/' + ':${PATH}'}

    factory.addStep(ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv"))
    factory.addStep(ShellCommand(command=['pip', 'install', '-U', 'wheel'], env=sandbox_path_top, haltOnFailure=True, workdir=".", name="Install wheels"))
    for repo_name, git_branch, _, _ in REPOS:
        repo_url = github_repo_url(repo_name)
        workdir = repo_name
        factory.addStep(
            Git(repourl=repo_url, branch=git_branch, workdir=workdir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url))
        if 'soledad' in repo_name:
            for subpackage in ["common", "client", "server"]:
                factory.addStep(
                    ShellCommand(command=generate_wheels, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="wheels for " + repo_name+"."+subpackage))
        else:
            factory.addStep(
                ShellCommand(command=generate_wheels, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="wheels for " + repo_name))
    factory.addStep(ShellCommand(command=publish_wheels, env=sandbox_path, doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))), workdir=".", name="publish wheels"))

    add_pyside_setup_repo(factory)

    return BuilderConfig(name=builder_name, slavenames=slaves.names(), factory=factory)

def publish_wheels_command():
    original_wheelhouse = config.get('ftp', 'copy_wheels_from')
    directory = config.get('ftp', 'directory')

    return ftp_publish_dir_command(original_wheelhouse, directory)

def add_pyside_setup_repo(factory):
    repo_name = "pyside-setup"
    repo_url = "https://github.com/ivanalejandro0/" + repo_name + ".git"
    git_branch = "master"
    
    venv_name = "virtualenv_wheels"
    sandbox_path = {'PATH':  "../" + venv_name + '/bin' + ':${PATH}'}

    publish_pyside_wheel = publish_pyside_command('`ls -t *.whl | head -1`')
    factory.addSteps([
        ShellCommand(command=['rm', '-rf', repo_name], workdir='.', env=sandbox_path, name="Remove previous pyside"),
        Git(repourl=repo_url, branch=git_branch, workdir=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url),
        ShellCommand(command=['python', 'setup.py', 'bdist_wheel', '--standalone'], workdir=repo_name, env=sandbox_path, name="Wheel for " + repo_name),
        ShellCommand(command=publish_pyside_wheel, workdir=repo_name + '/dist/', name="Publish pyside")
    ])

def publish_pyside_command(location):
    directory = config.get('ftp', 'directory')
    command = ftp_publish_command(location, directory)

    return command

def make_bundler_builder():
    builder_name = "builder_bundler"
    factory = BuildFactory()
    repo_name = "bitmask_bundler"
    repo_url = "https://github.com/leapcode/" + repo_name + ".git"
    branch = "develop"

    workdir="build"
    repo_dir = workdir + "/" + repo_name
    bundler_output_dir = "bundler_output"
    sumo_tarball = "leap.bitmask-latest-SUMO.tar.gz"

    publish_bundle = publish_bundle_command('`ls -t *.tar.gz | head -1`')

    factory.addSteps([
        Git(repourl=repo_url, branch=branch, workdir=repo_dir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url),
        ShellCommand(command="rm -rf " + bundler_output_dir, workdir=workdir, name="Remove previous bundler dir"),
        ShellCommand(command="mkdir " + bundler_output_dir, workdir=workdir, name="Create bundler dir"),
        ShellCommand(command="cp bundle_pyinstaller.sh ../" + bundler_output_dir, workdir=repo_dir, haltOnFailure=True, name="Copy bundle_pyinstaller"),
        ShellCommand(command="mkdir files", workdir=workdir + '/' + bundler_output_dir, name="Create auxiliary folder"),
        ShellCommand(command="wget http://lizard.leap.se/sumo-tarball/" + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, haltOnFailure=True, name="Download sumo"),
        ShellCommand(command="./bundle_pyinstaller.sh " + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, name="Create bundle"),
        ShellCommand(command=publish_bundle, workdir=workdir + '/' + bundler_output_dir, name="Publish bundle")
    ])

    return BuilderConfig(name=builder_name, slavenames=slaves.leap_names(), factory=factory)

def publish_bundle_command(location):
    directory = config.get('ftp', 'bundle_target_directory')
    command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'bitmask-latest.tar.gz')

    return command

c['builders'] = []

for repo_name, _, _, _ in REPOS:
    c['builders'].append(create_builder(repo_name))

c['builders'].append(make_wheel_builder())
c['builders'].append(make_bundler_builder())
####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

webinterface_username = "web_interface"
c['www'] = dict(port=PORT_WEB,
                plugins=dict(waterfall_view={}, console_view={}),
                change_hook_dialects={'github': { }})

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = config.get('Buildbot', 'title')
c['titleURL'] = "https://github.com/leapcode/bitmask_client"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = config.get('Buildbot', 'url')

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : "sqlite:///state.sqlite",
}