diff options
Diffstat (limited to 'master.cfg')
| -rw-r--r-- | master.cfg | 281 |
1 files changed, 7 insertions, 274 deletions
@@ -35,9 +35,7 @@ c['protocols'] = {'pb': {'port': PORT_MASTER}} # the 'change_source' setting tells the buildmaster how it should find out # about source code changes. -github_repos_username = 'leapcode' default_branch = 'develop' -order_repos_index = 2 # XXX: leap_mx doesn't need bitmask_client or leap_mail, not sure if two # pipelines here are worth it REPOS=[ @@ -49,9 +47,7 @@ REPOS=[ ('leap_mx', default_branch, 6, 'leap.mx'), ] -def github_repo_url(repo_name): - return 'https://github.com/' + github_repos_username + '/' + repo_name + '.git' - +from util import github_repo_url, order_repos_index all_repositories = { r'https://github.com/leapcode/leap_pycommon.git': 'leap_pycommon', @@ -145,281 +141,18 @@ c['schedulers'].append( # what steps, and which slaves can execute them. Note that any particular build will # only take place on one slave. -from buildbot.process.factory import BuildFactory -from buildbot.steps.source.git import Git -from buildbot.steps.shell import ShellCommand -from buildbot.config import BuilderConfig - -def add_repo_to_factory(factory, repo_name, git_branch, namespace, venv_name): - install_requirements = 'pkg/pip_install_requirements.sh --use-leap-wheels' - install_requirements_tests = "if [ -f pkg/requirements-testing.pip ]; then pkg/pip_install_requirements.sh --testing --use-leap-wheels; fi" - install = "python setup.py develop" - - workdir = repo_name - sandbox_path = {'PATH': "../" + venv_name + '/bin/' + ':${PATH}'} - sandbox_path_soledad = {'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} - repo_url = github_repo_url(repo_name) - - factory.addStep( - Git(repourl=repo_url, branch=git_branch, workdir=workdir, codebase=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url)) - - if 'bitmask_client' in repo_name: - factory.addSteps([ - ShellCommand(command='pkg/postmkvenv.sh',env=sandbox_path,haltOnFailure=False, workdir=workdir, name="postmkenv"), - ShellCommand(command='make',env=sandbox_path,haltOnFailure=False, workdir=workdir, name="make") - ]) - if 'soledad.git' in repo_url: - for subpackage in ["common", "client", "server"]: - if 'keymanager' in venv_name and subpackage is not "server" or 'keymanager' is not venv_name: # keymanager doesn't need soledad.server - factory.addSteps([ - ShellCommand(command=install_requirements, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="reqs: " + repo_name+"."+subpackage), - ShellCommand(command=install_requirements_tests, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="test reqs: " + repo_name+"."+subpackage), - ShellCommand(command=install, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="Install " + repo_name+"."+subpackage) - ]) - else: - factory.addSteps([ - ShellCommand(command=install_requirements, env=sandbox_path, haltOnFailure=False, workdir=workdir, name="reqs: " + repo_name), - ShellCommand(command=install_requirements_tests, env=sandbox_path, haltOnFailure=False, workdir=workdir, name="test reqs: " + repo_name), - ShellCommand(command=install, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="Install " + repo_name) - ]) - -def create_builder(repo_name): - builder_name = 'builder_' + repo_name - venv_name = "virtualenv_ci_" + builder_name - venv_path = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} - venv_path_factory = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} - - factory = BuildFactory() - factory.addSteps([ - ShellCommand(command=["rm", "-rf", venv_name], haltOnFailure=True, workdir=".", name="Remove previous virtualenv"), - ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv"), - ShellCommand(command=['pip', 'install', '-U', 'pip', 'setuptools', 'coverage'], env=venv_path, workdir=".", name="Update setuptools") - ]) +from builders import Builders - repo_index = [repo[order_repos_index] for repo in REPOS if repo[0] is repo_name][0] - for repo_name, repo_branch, _, namespace, in sorted(REPOS, key = lambda repo: repo[order_repos_index])[0:repo_index]: - add_repo_to_factory(factory, repo_name, repo_branch, namespace, venv_name) - - factory.addSteps([ - ShellCommand(command=['pep8', '.'],env=venv_path_factory,haltOnFailure=False, workdir=repo_name, name="pep8 on " + repo_name)]) - - if namespace is not '': - if repo_name is 'bitmask_client': - factory.addStep( - ShellCommand(command=['xvfb-run', 'coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) - else: - factory.addStep( - ShellCommand(command=['coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) - - factory.addSteps([ - ShellCommand(command=['coverage', 'html'], env=venv_path, workdir='.', name="generate html coverage report for " +namespace), - ShellCommand(command=publish_coverage_reports_command('htmlcov', repo_name), workdir='.', doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename')))) - ]) - - publish_leap_wheels(factory, repo_name, venv_path_factory, doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename')))) - - if repo_name == 'bitmask_client': - publish_sumo = publish_sumo_command('`ls -t *SUMO.tar.gz | head -1`') - - factory.addSteps([ - ShellCommand(command=['make', 'sumo_tarball_latest'], - env=venv_path_factory, workdir=repo_name, - doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))), - name="make sumo tarball"), - ShellCommand(command=publish_sumo, - env=venv_path_factory, workdir=repo_name + "/dist", - doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))), - name="publish sumo to ftp") - ]) - - - return BuilderConfig(name=builder_name, slavenames=slaves.names(), factory=factory) - -def publish_coverage_reports_command(location, repo_name): - target_directory = config.get('ftp', 'coverage_reports_target_directory') + '/' + repo_name + '_' + '`git -C ' + repo_name + ' describe`' - - return ftp_publish_dir_command(location, target_directory) - -def publish_leap_wheels(factory, repo_name, env, doStepIf): - env_soledad = {'PATH': env['PATH'].replace('../', '../../', 1)} - - if repo_name == 'soledad': - for subpackage in ["common", "client", "server"]: - factory.addSteps([ - ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage, name="leap wheels for " + repo_name+"."+subpackage), - ShellCommand(command=publish_leap_wheels_soledad(subpackage, '`ls -t *.whl | head -1`'), env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage+'/dist', name="publish leap wheels for " + repo_name+"."+subpackage)]) - else: - factory.addSteps([ - ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env, doStepIf=doStepIf, workdir=repo_name, name="Generate leap wheels for "+repo_name), - ShellCommand(command=publish_leap_wheels_command(repo_name, '`ls -t *.whl | head -1`'), env=env, doStepIf=doStepIf, workdir=repo_name + '/dist', name="Publish leap wheels for "+repo_name) - ]) - -def publish_leap_wheels_command(repo_name, location): - directory = config.get('ftp', 'leap_wheels_directory') - command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.' + repo_name + '-latest.whl') - - return command - -def publish_leap_wheels_soledad(subpackage, location): - directory = config.get('ftp', 'leap_wheels_directory') - command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.soledad.' + subpackage + '-latest.whl') - - return command - -def publish_sumo_command(location): - directory = config.get('ftp', 'sumo_target_directory') - command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.bitmask-latest-SUMO.tar.gz') - - return command - -def ftp_soft_link(filename, target_directory, symlink_name): - return ftp_ssh_command('ln -sf ' + target_directory + '/' + filename + ' ' + target_directory + '/' + symlink_name) - -def ftp_ssh_command(command): - ssh_port = config.get('ftp', 'ssh_port') - ssh_key = config.get('ftp', 'ssh_key') - user = config.get('ftp', 'user') - server = config.get('ftp', 'server') - - ssh_command = ['ssh', - "-i", ssh_key, - '-p', ssh_port, - user + '@' + server, - '"' + command + '"'] - - # Flatten to a string so that a shell executes de command, and - # expands ~ - return ' '.join(ssh_command) - -def ftp_publish_dir_command(from_dir, to_dir): - return ftp_publish_command(from_dir + "/*", to_dir) - -def ftp_publish_command(from_location, to_location): - ssh_port = config.get('ftp', 'ssh_port') - ssh_key = config.get('ftp', 'ssh_key') - user = config.get('ftp', 'user') - server = config.get('ftp', 'server') - - ssh_mkdir_command = ['ssh', - "-i", ssh_key, - '-p', ssh_port, - user + '@' + server, - '"mkdir ' + to_location + '"'] - - scp_command = ['scp', - '-i', ssh_key, - '-P', ssh_port, - '-r', from_location, - '"' + user + '@' + server + ':' + to_location +'"'] - ssh_command = ['ssh', - "-i", ssh_key, - '-p', ssh_port, - user + '@' + server, - '"chmod -R g+r ' + to_location + ' && chown -R ' + user + ':www-data ' + to_location + '"'] - # Flatten to a string so that a shell executes de command, and - # expands ~ - return ' '.join(ssh_mkdir_command) + ' ; ' + ' '.join(scp_command) + ' && ' + ' '.join(ssh_command) - -def make_wheel_builder(): - builder_name = "builder_wheels" - venv_name = "virtualenv_wheels" - factory = BuildFactory() - - generate_wheels = 'pkg/generate_wheels.sh' - publish_wheels = publish_wheels_command() - - sandbox_path_top = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} - sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} - sandbox_path_soledad = {'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} - - factory.addStep(ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv")) - factory.addStep(ShellCommand(command=['pip', 'install', '-U', 'wheel'], env=sandbox_path_top, haltOnFailure=True, workdir=".", name="Install wheels")) - for repo_name, git_branch, _, _ in REPOS: - repo_url = github_repo_url(repo_name) - workdir = repo_name - factory.addStep( - Git(repourl=repo_url, branch=git_branch, workdir=workdir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url)) - if 'soledad' in repo_name: - for subpackage in ["common", "client", "server"]: - factory.addStep( - ShellCommand(command=generate_wheels, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="wheels for " + repo_name+"."+subpackage)) - else: - factory.addStep( - ShellCommand(command=generate_wheels, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="wheels for " + repo_name)) - factory.addStep(ShellCommand(command=publish_wheels, env=sandbox_path, doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))), workdir=".", name="publish wheels")) - - add_pyside_setup_repo(factory) - - return BuilderConfig(name=builder_name, slavenames=slaves.names(), factory=factory) - -def publish_wheels_command(): - original_wheelhouse = config.get('ftp', 'copy_wheels_from') - directory = config.get('ftp', 'directory') - - return ftp_publish_dir_command(original_wheelhouse, directory) - -def add_pyside_setup_repo(factory): - repo_name = "pyside-setup" - repo_url = "https://github.com/ivanalejandro0/" + repo_name + ".git" - git_branch = "master" - - venv_name = "virtualenv_wheels" - sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} - - publish_pyside_wheel = publish_pyside_command('`ls -t *.whl | head -1`') - factory.addSteps([ - ShellCommand(command=['rm', '-rf', repo_name], workdir='.', env=sandbox_path, name="Remove previous pyside"), - Git(repourl=repo_url, branch=git_branch, workdir=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), - ShellCommand(command=['python', 'setup.py', 'bdist_wheel', '--standalone'], workdir=repo_name, env=sandbox_path, name="Wheel for " + repo_name), - ShellCommand(command=publish_pyside_wheel, workdir=repo_name + '/dist/', name="Publish pyside") - ]) - -def publish_pyside_command(location): - directory = config.get('ftp', 'directory') - command = ftp_publish_command(location, directory) - - return command - -def make_bundler_builder(): - builder_name = "builder_bundler" - factory = BuildFactory() - repo_name = "bitmask_bundler" - repo_url = "https://github.com/leapcode/" + repo_name + ".git" - branch = "develop" - - workdir="build" - repo_dir = workdir + "/" + repo_name - bundler_output_dir = "bundler_output" - sumo_tarball = "leap.bitmask-latest-SUMO.tar.gz" - - publish_bundle = publish_bundle_command('`ls -t *.tar.gz | head -1`') - - factory.addSteps([ - Git(repourl=repo_url, branch=branch, workdir=repo_dir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), - ShellCommand(command="rm -rf " + bundler_output_dir, workdir=workdir, name="Remove previous bundler dir"), - ShellCommand(command="mkdir " + bundler_output_dir, workdir=workdir, name="Create bundler dir"), - ShellCommand(command="cp bundle_pyinstaller.sh ../" + bundler_output_dir, workdir=repo_dir, haltOnFailure=True, name="Copy bundle_pyinstaller"), - ShellCommand(command="mkdir files", workdir=workdir + '/' + bundler_output_dir, name="Create auxiliary folder"), - ShellCommand(command="wget http://lizard.leap.se/sumo-tarball/" + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, haltOnFailure=True, name="Download sumo"), - ShellCommand(command="./bundle_pyinstaller.sh " + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, name="Create bundle"), - ShellCommand(command=publish_bundle, workdir=workdir + '/' + bundler_output_dir, name="Publish bundle") - ]) - - return BuilderConfig(name=builder_name, slavenames=slaves.leap_names(), factory=factory) - -def publish_bundle_command(location): - directory = config.get('ftp', 'bundle_target_directory') - command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'bitmask-latest.tar.gz') - - return command +builders = Builders(config, slaves, REPOS) c['builders'] = [] for repo_name, _, _, _ in REPOS: - c['builders'].append(create_builder(repo_name)) + c['builders'].append(builders.create_builder(repo_name)) + +c['builders'].append(builders.make_wheel_builder()) +c['builders'].append(builders.make_bundler_builder()) -c['builders'].append(make_wheel_builder()) -c['builders'].append(make_bundler_builder()) ####### STATUS TARGETS # 'status' is a list of Status Targets. The results of each build will be |