From 2fffd4413ea2b9785cd8feb254207ed1366b3a15 Mon Sep 17 00:00:00 2001 From: Ruben Pollan Date: Wed, 25 Nov 2015 01:15:46 +0100 Subject: [style] Refactor builders code to it's own file --- builders.py | 321 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ master.cfg | 281 ++-------------------------------------------------- util.py | 7 ++ 3 files changed, 335 insertions(+), 274 deletions(-) create mode 100644 builders.py create mode 100644 util.py diff --git a/builders.py b/builders.py new file mode 100644 index 0000000..35cd9b3 --- /dev/null +++ b/builders.py @@ -0,0 +1,321 @@ +from util import github_repo_url, order_repos_index + +from buildbot.process.factory import BuildFactory +from buildbot.steps.source.git import Git +from buildbot.steps.shell import ShellCommand +from buildbot.config import BuilderConfig + + +class Builders(object): + + def __init__(self, config, slaves, repos): + self.config = config + self.slaves = slaves + self.repos = repos + + def create_builder(self, repo_name): + builder_name = 'builder_' + repo_name + venv_name = "virtualenv_ci_" + builder_name + venv_path = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} + venv_path_factory = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} + + factory = BuildFactory() + factory.addSteps([ + ShellCommand(command=["rm", "-rf", venv_name], haltOnFailure=True, + workdir=".", name="Remove previous virtualenv"), + ShellCommand(command=["virtualenv", "--python=python2", venv_name], + haltOnFailure=True, workdir=".", + name="Create new virtualenv"), + ShellCommand( + command=['pip', 'install', '-U', 'pip', 'setuptools', + 'coverage'], + env=venv_path, workdir=".", name="Update setuptools") + ]) + + repo_index = [repo[order_repos_index] for repo in self.repos if repo[0] is repo_name][0] + for repo_name, repo_branch, _, namespace, in sorted(self.repos, key = lambda repo: repo[order_repos_index])[0:repo_index]: + _add_repo_to_factory(factory, repo_name, repo_branch, namespace, venv_name) + + factory.addSteps([ + ShellCommand(command=['pep8', '.'],env=venv_path_factory,haltOnFailure=False, workdir=repo_name, name="pep8 on " + repo_name)]) + + if namespace is not '': + if repo_name is 'bitmask_client': + factory.addStep( + ShellCommand(command=['xvfb-run', 'coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) + else: + factory.addStep( + ShellCommand(command=['coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) + + factory.addSteps([ + ShellCommand(command=['coverage', 'html'], env=venv_path, workdir='.', name="generate html coverage report for " +namespace), + ShellCommand(command=self._publish_coverage_reports_command('htmlcov', repo_name), workdir='.', doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename')))) + ]) + + self._publish_leap_wheels(factory, repo_name, venv_path_factory, doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename')))) + + if repo_name == 'bitmask_client': + publish_sumo = self._publish_sumo_command('`ls -t *SUMO.tar.gz | head -1`') + + factory.addSteps([ + ShellCommand(command=['make', 'sumo_tarball_latest'], + env=venv_path_factory, workdir=repo_name, + doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), + name="make sumo tarball"), + ShellCommand(command=publish_sumo, + env=venv_path_factory, workdir=repo_name + "/dist", + doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), + name="publish sumo to ftp") + ]) + + + return BuilderConfig(name=builder_name, slavenames=self.slaves.names(), factory=factory) + + def _publish_coverage_reports_command(self, location, repo_name): + target_directory = self.config.get('ftp', 'coverage_reports_target_directory') + '/' + repo_name + '_' + '`git -C ' + repo_name + ' describe`' + + return self._ftp_publish_dir_command(location, target_directory) + + def _publish_leap_wheels(self, factory, repo_name, env, doStepIf): + env_soledad = {'PATH': env['PATH'].replace('../', '../../', 1)} + + if repo_name == 'soledad': + for subpackage in ["common", "client", "server"]: + factory.addSteps([ + ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage, name="leap wheels for " + repo_name+"."+subpackage), + ShellCommand(command=self._publish_leap_wheels_soledad(subpackage, '`ls -t *.whl | head -1`'), env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage+'/dist', name="publish leap wheels for " + repo_name+"."+subpackage)]) + else: + factory.addSteps([ + ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env, doStepIf=doStepIf, workdir=repo_name, name="Generate leap wheels for "+repo_name), + ShellCommand(command=self._publish_leap_wheels_command(repo_name, '`ls -t *.whl | head -1`'), env=env, doStepIf=doStepIf, workdir=repo_name + '/dist', name="Publish leap wheels for "+repo_name) + ]) + + def _publish_leap_wheels_command(self, repo_name, location): + directory = self.config.get('ftp', 'leap_wheels_directory') + command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.' + repo_name + '-latest.whl') + + return command + + def _publish_leap_wheels_soledad(self, subpackage, location): + directory = self.config.get('ftp', 'leap_wheels_directory') + command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.soledad.' + subpackage + '-latest.whl') + + return command + + def _publish_sumo_command(self, location): + directory = self.config.get('ftp', 'sumo_target_directory') + command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.bitmask-latest-SUMO.tar.gz') + + return command + + def _ftp_soft_link(self, filename, target_directory, symlink_name): + return self._ftp_ssh_command('ln -sf ' + target_directory + '/' + filename + ' ' + target_directory + '/' + symlink_name) + + def _ftp_ssh_command(self, command): + ssh_port = self.config.get('ftp', 'ssh_port') + ssh_key = self.config.get('ftp', 'ssh_key') + user = self.config.get('ftp', 'user') + server = self.config.get('ftp', 'server') + + ssh_command = ['ssh', + "-i", ssh_key, + '-p', ssh_port, + user + '@' + server, + '"' + command + '"'] + + # Flatten to a string so that a shell executes de command, and + # expands ~ + return ' '.join(ssh_command) + + def _ftp_publish_dir_command(self, from_dir, to_dir): + return self._ftp_publish_command(from_dir + "/*", to_dir) + + def _ftp_publish_command(self, from_location, to_location): + ssh_port = self.config.get('ftp', 'ssh_port') + ssh_key = self.config.get('ftp', 'ssh_key') + user = self.config.get('ftp', 'user') + server = self.config.get('ftp', 'server') + + ssh_mkdir_command = ['ssh', + "-i", ssh_key, + '-p', ssh_port, + user + '@' + server, + '"mkdir ' + to_location + '"'] + + scp_command = ['scp', + '-i', ssh_key, + '-P', ssh_port, + '-r', from_location, + '"' + user + '@' + server + ':' + to_location +'"'] + ssh_command = ['ssh', + "-i", ssh_key, + '-p', ssh_port, + user + '@' + server, + '"chmod -R g+r ' + to_location + ' && chown -R ' + user + ':www-data ' + to_location + '"'] + # Flatten to a string so that a shell executes de command, and + # expands ~ + return ' '.join(ssh_mkdir_command) + ' ; ' + ' '.join(scp_command) + ' && ' + ' '.join(ssh_command) + + def make_wheel_builder(self): + builder_name = "builder_wheels" + venv_name = "virtualenv_wheels" + factory = BuildFactory() + + generate_wheels = 'pkg/generate_wheels.sh' + publish_wheels = self._publish_wheels_command() + + sandbox_path_top = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} + sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} + sandbox_path_soledad = {'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} + + factory.addStep(ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv")) + factory.addStep(ShellCommand(command=['pip', 'install', '-U', 'wheel'], env=sandbox_path_top, haltOnFailure=True, workdir=".", name="Install wheels")) + for repo_name, git_branch, _, _ in self.repos: + repo_url = github_repo_url(repo_name) + workdir = repo_name + factory.addStep( + Git(repourl=repo_url, branch=git_branch, workdir=workdir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url)) + if 'soledad' in repo_name: + for subpackage in ["common", "client", "server"]: + factory.addStep( + ShellCommand(command=generate_wheels, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="wheels for " + repo_name+"."+subpackage)) + else: + factory.addStep( + ShellCommand(command=generate_wheels, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="wheels for " + repo_name)) + factory.addStep(ShellCommand(command=publish_wheels, env=sandbox_path, doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), workdir=".", name="publish wheels")) + + self._add_pyside_setup_repo(factory) + + return BuilderConfig(name=builder_name, slavenames=self.slaves.names(), factory=factory) + + def _publish_wheels_command(self): + original_wheelhouse = self.config.get('ftp', 'copy_wheels_from') + directory = self.config.get('ftp', 'directory') + + return self._ftp_publish_dir_command(original_wheelhouse, directory) + + def _add_pyside_setup_repo(self, factory): + repo_name = "pyside-setup" + repo_url = "https://github.com/ivanalejandro0/" + repo_name + ".git" + git_branch = "master" + + venv_name = "virtualenv_wheels" + sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} + + publish_pyside_wheel = self._publish_pyside_command('`ls -t *.whl | head -1`') + factory.addSteps([ + ShellCommand(command=['rm', '-rf', repo_name], workdir='.', env=sandbox_path, name="Remove previous pyside"), + Git(repourl=repo_url, branch=git_branch, workdir=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), + ShellCommand(command=['python', 'setup.py', 'bdist_wheel', '--standalone'], workdir=repo_name, env=sandbox_path, name="Wheel for " + repo_name), + ShellCommand(command=publish_pyside_wheel, workdir=repo_name + '/dist/', name="Publish pyside") + ]) + + def _publish_pyside_command(self, location): + directory = self.config.get('ftp', 'directory') + command = self._ftp_publish_command(location, directory) + + return command + + def make_bundler_builder(self): + builder_name = "builder_bundler" + factory = BuildFactory() + repo_name = "bitmask_bundler" + repo_url = "https://github.com/leapcode/" + repo_name + ".git" + branch = "develop" + + workdir="build" + repo_dir = workdir + "/" + repo_name + bundler_output_dir = "bundler_output" + sumo_tarball = "leap.bitmask-latest-SUMO.tar.gz" + + publish_bundle = self._publish_bundle_command('`ls -t *.tar.gz | head -1`') + + factory.addSteps([ + Git(repourl=repo_url, branch=branch, workdir=repo_dir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), + ShellCommand(command="rm -rf " + bundler_output_dir, workdir=workdir, name="Remove previous bundler dir"), + ShellCommand(command="mkdir " + bundler_output_dir, workdir=workdir, name="Create bundler dir"), + ShellCommand(command="cp bundle_pyinstaller.sh ../" + bundler_output_dir, workdir=repo_dir, haltOnFailure=True, name="Copy bundle_pyinstaller"), + ShellCommand(command="mkdir files", workdir=workdir + '/' + bundler_output_dir, name="Create auxiliary folder"), + ShellCommand(command="wget http://lizard.leap.se/sumo-tarball/" + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, haltOnFailure=True, name="Download sumo"), + ShellCommand(command="./bundle_pyinstaller.sh " + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, name="Create bundle"), + ShellCommand(command=publish_bundle, workdir=workdir + '/' + bundler_output_dir, name="Publish bundle") + ]) + + return BuilderConfig(name=builder_name, slavenames=self.slaves.leap_names(), factory=factory) + + def _publish_bundle_command(self, location): + directory = self.config.get('ftp', 'bundle_target_directory') + command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'bitmask-latest.tar.gz') + + return command + + +def _add_repo_to_factory(factory, repo_name, git_branch, namespace, venv_name): + install_requirements = ( + "pkg/pip_install_requirements.sh --use-leap-wheels") + install_requirements_tests = """ + if [ -f pkg/requirements-testing.pip ] + then + pkg/pip_install_requirements.sh --testing --use-leap-wheels + fi + """ + install = "python setup.py develop" + + workdir = repo_name + sandbox_path = {'PATH': "../" + venv_name + '/bin/' + ':${PATH}'} + sandbox_path_soledad = { + 'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} + repo_url = github_repo_url(repo_name) + + factory.addStep( + Git(repourl=repo_url, branch=git_branch, workdir=workdir, + codebase=repo_name, mode='full', method='clobber', + shallow=True, haltOnFailure=True, name="Pull " + repo_url)) + + if 'bitmask_client' in repo_name: + factory.addSteps([ + ShellCommand(command='pkg/postmkvenv.sh', env=sandbox_path, + haltOnFailure=False, workdir=workdir, + name="postmkenv"), + ShellCommand(command='make', env=sandbox_path, + haltOnFailure=False, workdir=workdir, + name="make") + ]) + if 'soledad.git' in repo_url: + for subpackage in ["common", "client", "server"]: + # keymanager doesn't need soledad.server + if ('keymanager' in venv_name and + subpackage is not "server" or + 'keymanager' is not venv_name): + factory.addSteps([ + ShellCommand( + command=install_requirements, + env=sandbox_path_soledad, + haltOnFailure=True, + workdir=workdir+'/'+subpackage, + name="reqs: " + repo_name+"."+subpackage), + ShellCommand( + command=install_requirements_tests, + env=sandbox_path_soledad, + haltOnFailure=True, + workdir=workdir+'/'+subpackage, + name="test reqs: " + repo_name+"."+subpackage), + ShellCommand( + command=install, + env=sandbox_path_soledad, + haltOnFailure=True, + workdir=workdir+'/'+subpackage, + name="Install " + repo_name+"."+subpackage) + ]) + else: + factory.addSteps([ + ShellCommand(command=install_requirements, env=sandbox_path, + haltOnFailure=False, workdir=workdir, + name="reqs: " + repo_name), + ShellCommand(command=install_requirements_tests, + env=sandbox_path, haltOnFailure=False, + workdir=workdir, name="test reqs: " + repo_name), + ShellCommand(command=install, env=sandbox_path, + haltOnFailure=True, workdir=workdir, + name="Install " + repo_name) + ]) diff --git a/master.cfg b/master.cfg index 82d8a5b..4c7c398 100644 --- a/master.cfg +++ b/master.cfg @@ -35,9 +35,7 @@ c['protocols'] = {'pb': {'port': PORT_MASTER}} # the 'change_source' setting tells the buildmaster how it should find out # about source code changes. -github_repos_username = 'leapcode' default_branch = 'develop' -order_repos_index = 2 # XXX: leap_mx doesn't need bitmask_client or leap_mail, not sure if two # pipelines here are worth it REPOS=[ @@ -49,9 +47,7 @@ REPOS=[ ('leap_mx', default_branch, 6, 'leap.mx'), ] -def github_repo_url(repo_name): - return 'https://github.com/' + github_repos_username + '/' + repo_name + '.git' - +from util import github_repo_url, order_repos_index all_repositories = { r'https://github.com/leapcode/leap_pycommon.git': 'leap_pycommon', @@ -145,281 +141,18 @@ c['schedulers'].append( # what steps, and which slaves can execute them. Note that any particular build will # only take place on one slave. -from buildbot.process.factory import BuildFactory -from buildbot.steps.source.git import Git -from buildbot.steps.shell import ShellCommand -from buildbot.config import BuilderConfig - -def add_repo_to_factory(factory, repo_name, git_branch, namespace, venv_name): - install_requirements = 'pkg/pip_install_requirements.sh --use-leap-wheels' - install_requirements_tests = "if [ -f pkg/requirements-testing.pip ]; then pkg/pip_install_requirements.sh --testing --use-leap-wheels; fi" - install = "python setup.py develop" - - workdir = repo_name - sandbox_path = {'PATH': "../" + venv_name + '/bin/' + ':${PATH}'} - sandbox_path_soledad = {'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} - repo_url = github_repo_url(repo_name) - - factory.addStep( - Git(repourl=repo_url, branch=git_branch, workdir=workdir, codebase=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url)) - - if 'bitmask_client' in repo_name: - factory.addSteps([ - ShellCommand(command='pkg/postmkvenv.sh',env=sandbox_path,haltOnFailure=False, workdir=workdir, name="postmkenv"), - ShellCommand(command='make',env=sandbox_path,haltOnFailure=False, workdir=workdir, name="make") - ]) - if 'soledad.git' in repo_url: - for subpackage in ["common", "client", "server"]: - if 'keymanager' in venv_name and subpackage is not "server" or 'keymanager' is not venv_name: # keymanager doesn't need soledad.server - factory.addSteps([ - ShellCommand(command=install_requirements, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="reqs: " + repo_name+"."+subpackage), - ShellCommand(command=install_requirements_tests, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="test reqs: " + repo_name+"."+subpackage), - ShellCommand(command=install, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="Install " + repo_name+"."+subpackage) - ]) - else: - factory.addSteps([ - ShellCommand(command=install_requirements, env=sandbox_path, haltOnFailure=False, workdir=workdir, name="reqs: " + repo_name), - ShellCommand(command=install_requirements_tests, env=sandbox_path, haltOnFailure=False, workdir=workdir, name="test reqs: " + repo_name), - ShellCommand(command=install, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="Install " + repo_name) - ]) - -def create_builder(repo_name): - builder_name = 'builder_' + repo_name - venv_name = "virtualenv_ci_" + builder_name - venv_path = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} - venv_path_factory = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} - - factory = BuildFactory() - factory.addSteps([ - ShellCommand(command=["rm", "-rf", venv_name], haltOnFailure=True, workdir=".", name="Remove previous virtualenv"), - ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv"), - ShellCommand(command=['pip', 'install', '-U', 'pip', 'setuptools', 'coverage'], env=venv_path, workdir=".", name="Update setuptools") - ]) +from builders import Builders - repo_index = [repo[order_repos_index] for repo in REPOS if repo[0] is repo_name][0] - for repo_name, repo_branch, _, namespace, in sorted(REPOS, key = lambda repo: repo[order_repos_index])[0:repo_index]: - add_repo_to_factory(factory, repo_name, repo_branch, namespace, venv_name) - - factory.addSteps([ - ShellCommand(command=['pep8', '.'],env=venv_path_factory,haltOnFailure=False, workdir=repo_name, name="pep8 on " + repo_name)]) - - if namespace is not '': - if repo_name is 'bitmask_client': - factory.addStep( - ShellCommand(command=['xvfb-run', 'coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) - else: - factory.addStep( - ShellCommand(command=['coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) - - factory.addSteps([ - ShellCommand(command=['coverage', 'html'], env=venv_path, workdir='.', name="generate html coverage report for " +namespace), - ShellCommand(command=publish_coverage_reports_command('htmlcov', repo_name), workdir='.', doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename')))) - ]) - - publish_leap_wheels(factory, repo_name, venv_path_factory, doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename')))) - - if repo_name == 'bitmask_client': - publish_sumo = publish_sumo_command('`ls -t *SUMO.tar.gz | head -1`') - - factory.addSteps([ - ShellCommand(command=['make', 'sumo_tarball_latest'], - env=venv_path_factory, workdir=repo_name, - doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))), - name="make sumo tarball"), - ShellCommand(command=publish_sumo, - env=venv_path_factory, workdir=repo_name + "/dist", - doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))), - name="publish sumo to ftp") - ]) - - - return BuilderConfig(name=builder_name, slavenames=slaves.names(), factory=factory) - -def publish_coverage_reports_command(location, repo_name): - target_directory = config.get('ftp', 'coverage_reports_target_directory') + '/' + repo_name + '_' + '`git -C ' + repo_name + ' describe`' - - return ftp_publish_dir_command(location, target_directory) - -def publish_leap_wheels(factory, repo_name, env, doStepIf): - env_soledad = {'PATH': env['PATH'].replace('../', '../../', 1)} - - if repo_name == 'soledad': - for subpackage in ["common", "client", "server"]: - factory.addSteps([ - ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage, name="leap wheels for " + repo_name+"."+subpackage), - ShellCommand(command=publish_leap_wheels_soledad(subpackage, '`ls -t *.whl | head -1`'), env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage+'/dist', name="publish leap wheels for " + repo_name+"."+subpackage)]) - else: - factory.addSteps([ - ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env, doStepIf=doStepIf, workdir=repo_name, name="Generate leap wheels for "+repo_name), - ShellCommand(command=publish_leap_wheels_command(repo_name, '`ls -t *.whl | head -1`'), env=env, doStepIf=doStepIf, workdir=repo_name + '/dist', name="Publish leap wheels for "+repo_name) - ]) - -def publish_leap_wheels_command(repo_name, location): - directory = config.get('ftp', 'leap_wheels_directory') - command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.' + repo_name + '-latest.whl') - - return command - -def publish_leap_wheels_soledad(subpackage, location): - directory = config.get('ftp', 'leap_wheels_directory') - command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.soledad.' + subpackage + '-latest.whl') - - return command - -def publish_sumo_command(location): - directory = config.get('ftp', 'sumo_target_directory') - command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'leap.bitmask-latest-SUMO.tar.gz') - - return command - -def ftp_soft_link(filename, target_directory, symlink_name): - return ftp_ssh_command('ln -sf ' + target_directory + '/' + filename + ' ' + target_directory + '/' + symlink_name) - -def ftp_ssh_command(command): - ssh_port = config.get('ftp', 'ssh_port') - ssh_key = config.get('ftp', 'ssh_key') - user = config.get('ftp', 'user') - server = config.get('ftp', 'server') - - ssh_command = ['ssh', - "-i", ssh_key, - '-p', ssh_port, - user + '@' + server, - '"' + command + '"'] - - # Flatten to a string so that a shell executes de command, and - # expands ~ - return ' '.join(ssh_command) - -def ftp_publish_dir_command(from_dir, to_dir): - return ftp_publish_command(from_dir + "/*", to_dir) - -def ftp_publish_command(from_location, to_location): - ssh_port = config.get('ftp', 'ssh_port') - ssh_key = config.get('ftp', 'ssh_key') - user = config.get('ftp', 'user') - server = config.get('ftp', 'server') - - ssh_mkdir_command = ['ssh', - "-i", ssh_key, - '-p', ssh_port, - user + '@' + server, - '"mkdir ' + to_location + '"'] - - scp_command = ['scp', - '-i', ssh_key, - '-P', ssh_port, - '-r', from_location, - '"' + user + '@' + server + ':' + to_location +'"'] - ssh_command = ['ssh', - "-i", ssh_key, - '-p', ssh_port, - user + '@' + server, - '"chmod -R g+r ' + to_location + ' && chown -R ' + user + ':www-data ' + to_location + '"'] - # Flatten to a string so that a shell executes de command, and - # expands ~ - return ' '.join(ssh_mkdir_command) + ' ; ' + ' '.join(scp_command) + ' && ' + ' '.join(ssh_command) - -def make_wheel_builder(): - builder_name = "builder_wheels" - venv_name = "virtualenv_wheels" - factory = BuildFactory() - - generate_wheels = 'pkg/generate_wheels.sh' - publish_wheels = publish_wheels_command() - - sandbox_path_top = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} - sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} - sandbox_path_soledad = {'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} - - factory.addStep(ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv")) - factory.addStep(ShellCommand(command=['pip', 'install', '-U', 'wheel'], env=sandbox_path_top, haltOnFailure=True, workdir=".", name="Install wheels")) - for repo_name, git_branch, _, _ in REPOS: - repo_url = github_repo_url(repo_name) - workdir = repo_name - factory.addStep( - Git(repourl=repo_url, branch=git_branch, workdir=workdir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url)) - if 'soledad' in repo_name: - for subpackage in ["common", "client", "server"]: - factory.addStep( - ShellCommand(command=generate_wheels, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="wheels for " + repo_name+"."+subpackage)) - else: - factory.addStep( - ShellCommand(command=generate_wheels, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="wheels for " + repo_name)) - factory.addStep(ShellCommand(command=publish_wheels, env=sandbox_path, doStepIf=(lambda step: slaves.is_leap(step.getProperty('slavename'))), workdir=".", name="publish wheels")) - - add_pyside_setup_repo(factory) - - return BuilderConfig(name=builder_name, slavenames=slaves.names(), factory=factory) - -def publish_wheels_command(): - original_wheelhouse = config.get('ftp', 'copy_wheels_from') - directory = config.get('ftp', 'directory') - - return ftp_publish_dir_command(original_wheelhouse, directory) - -def add_pyside_setup_repo(factory): - repo_name = "pyside-setup" - repo_url = "https://github.com/ivanalejandro0/" + repo_name + ".git" - git_branch = "master" - - venv_name = "virtualenv_wheels" - sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} - - publish_pyside_wheel = publish_pyside_command('`ls -t *.whl | head -1`') - factory.addSteps([ - ShellCommand(command=['rm', '-rf', repo_name], workdir='.', env=sandbox_path, name="Remove previous pyside"), - Git(repourl=repo_url, branch=git_branch, workdir=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), - ShellCommand(command=['python', 'setup.py', 'bdist_wheel', '--standalone'], workdir=repo_name, env=sandbox_path, name="Wheel for " + repo_name), - ShellCommand(command=publish_pyside_wheel, workdir=repo_name + '/dist/', name="Publish pyside") - ]) - -def publish_pyside_command(location): - directory = config.get('ftp', 'directory') - command = ftp_publish_command(location, directory) - - return command - -def make_bundler_builder(): - builder_name = "builder_bundler" - factory = BuildFactory() - repo_name = "bitmask_bundler" - repo_url = "https://github.com/leapcode/" + repo_name + ".git" - branch = "develop" - - workdir="build" - repo_dir = workdir + "/" + repo_name - bundler_output_dir = "bundler_output" - sumo_tarball = "leap.bitmask-latest-SUMO.tar.gz" - - publish_bundle = publish_bundle_command('`ls -t *.tar.gz | head -1`') - - factory.addSteps([ - Git(repourl=repo_url, branch=branch, workdir=repo_dir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), - ShellCommand(command="rm -rf " + bundler_output_dir, workdir=workdir, name="Remove previous bundler dir"), - ShellCommand(command="mkdir " + bundler_output_dir, workdir=workdir, name="Create bundler dir"), - ShellCommand(command="cp bundle_pyinstaller.sh ../" + bundler_output_dir, workdir=repo_dir, haltOnFailure=True, name="Copy bundle_pyinstaller"), - ShellCommand(command="mkdir files", workdir=workdir + '/' + bundler_output_dir, name="Create auxiliary folder"), - ShellCommand(command="wget http://lizard.leap.se/sumo-tarball/" + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, haltOnFailure=True, name="Download sumo"), - ShellCommand(command="./bundle_pyinstaller.sh " + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, name="Create bundle"), - ShellCommand(command=publish_bundle, workdir=workdir + '/' + bundler_output_dir, name="Publish bundle") - ]) - - return BuilderConfig(name=builder_name, slavenames=slaves.leap_names(), factory=factory) - -def publish_bundle_command(location): - directory = config.get('ftp', 'bundle_target_directory') - command = ftp_publish_command(location, directory) + ' && ' + ftp_soft_link(location, directory, 'bitmask-latest.tar.gz') - - return command +builders = Builders(config, slaves, REPOS) c['builders'] = [] for repo_name, _, _, _ in REPOS: - c['builders'].append(create_builder(repo_name)) + c['builders'].append(builders.create_builder(repo_name)) + +c['builders'].append(builders.make_wheel_builder()) +c['builders'].append(builders.make_bundler_builder()) -c['builders'].append(make_wheel_builder()) -c['builders'].append(make_bundler_builder()) ####### STATUS TARGETS # 'status' is a list of Status Targets. The results of each build will be diff --git a/util.py b/util.py new file mode 100644 index 0000000..3eb7cb8 --- /dev/null +++ b/util.py @@ -0,0 +1,7 @@ +github_repos_username = 'leapcode' +order_repos_index = 2 + + +def github_repo_url(repo_name): + return ('https://github.com/' + github_repos_username + '/' + repo_name + + '.git') -- cgit v1.2.3