from util import github_repo_url, order_repos_index from buildbot.process.factory import BuildFactory from buildbot.steps.source.git import Git from buildbot.steps.shell import ShellCommand from buildbot.config import BuilderConfig class Builders(object): def __init__(self, config, slaves, repos): self.config = config self.slaves = slaves self.repos = repos def create_builder(self, repo_name): builder_name = 'builder_' + repo_name venv_name = "virtualenv_ci_" + builder_name venv_path = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} venv_path_factory = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} factory = BuildFactory() factory.addSteps([ ShellCommand(command=["rm", "-rf", venv_name], haltOnFailure=True, workdir=".", name="Remove previous virtualenv"), ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv"), ShellCommand( command=['pip', 'install', '-U', 'pip', 'setuptools', 'coverage'], env=venv_path, workdir=".", name="Update setuptools") ]) repo_index = [repo[order_repos_index] for repo in self.repos if repo[0] is repo_name][0] for repo_name, repo_branch, _, namespace, in sorted(self.repos, key = lambda repo: repo[order_repos_index])[0:repo_index]: _add_repo_to_factory(factory, repo_name, repo_branch, namespace, venv_name) factory.addSteps([ ShellCommand(command=['pep8', '.'],env=venv_path_factory,haltOnFailure=False, workdir=repo_name, name="pep8 on " + repo_name)]) if namespace is not '': if repo_name is 'bitmask_client': factory.addStep( ShellCommand(command=['xvfb-run', 'coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/nosetests', namespace], env=venv_path, workdir='.', name="nosetests "+namespace)) else: factory.addStep( ShellCommand(command=['coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) factory.addSteps([ ShellCommand(command=['coverage', 'html'], env=venv_path, workdir='.', name="generate html coverage report for " +namespace), ShellCommand(command=self._publish_coverage_reports_command('htmlcov', repo_name), workdir='.', doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename')))) ]) self._publish_leap_wheels(factory, repo_name, venv_path_factory, doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename')))) if repo_name == 'bitmask_client': publish_sumo = self._publish_sumo_command('`ls -t *SUMO.tar.gz | head -1`') factory.addSteps([ ShellCommand(command=['make', 'sumo_tarball_latest'], env=venv_path_factory, workdir=repo_name, doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), name="make sumo tarball"), ShellCommand(command=publish_sumo, env=venv_path_factory, workdir=repo_name + "/dist", doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), name="publish sumo to ftp") ]) return BuilderConfig(name=builder_name, slavenames=self.slaves.names(), factory=factory) def _publish_coverage_reports_command(self, location, repo_name): target_directory = self.config.get('ftp', 'coverage_reports_target_directory') + '/' + repo_name + '_' + '`git -C ' + repo_name + ' describe`' return self._ftp_publish_dir_command(location, target_directory) def _publish_leap_wheels(self, factory, repo_name, env, doStepIf): env_soledad = {'PATH': env['PATH'].replace('../', '../../', 1)} if repo_name == 'soledad': for subpackage in ["common", "client", "server"]: factory.addSteps([ ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage, name="leap wheels for " + repo_name+"."+subpackage), ShellCommand(command=self._publish_leap_wheels_soledad(subpackage, '`ls -t *.whl | head -1`'), env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage+'/dist', name="publish leap wheels for " + repo_name+"."+subpackage)]) else: factory.addSteps([ ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env, doStepIf=doStepIf, workdir=repo_name, name="Generate leap wheels for "+repo_name), ShellCommand(command=self._publish_leap_wheels_command(repo_name, '`ls -t *.whl | head -1`'), env=env, doStepIf=doStepIf, workdir=repo_name + '/dist', name="Publish leap wheels for "+repo_name) ]) def _publish_leap_wheels_command(self, repo_name, location): directory = self.config.get('ftp', 'leap_wheels_directory') command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.' + repo_name + '-latest.whl') return command def _publish_leap_wheels_soledad(self, subpackage, location): directory = self.config.get('ftp', 'leap_wheels_directory') command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.soledad.' + subpackage + '-latest.whl') return command def _publish_sumo_command(self, location): directory = self.config.get('ftp', 'sumo_target_directory') command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.bitmask-latest-SUMO.tar.gz') return command def _ftp_soft_link(self, filename, target_directory, symlink_name): return self._ftp_ssh_command('ln -sf ' + target_directory + '/' + filename + ' ' + target_directory + '/' + symlink_name) def _ftp_ssh_command(self, command): ssh_port = self.config.get('ftp', 'ssh_port') ssh_key = self.config.get('ftp', 'ssh_key') user = self.config.get('ftp', 'user') server = self.config.get('ftp', 'server') ssh_command = ['ssh', "-i", ssh_key, '-p', ssh_port, user + '@' + server, '"' + command + '"'] # Flatten to a string so that a shell executes de command, and # expands ~ return ' '.join(ssh_command) def _ftp_publish_dir_command(self, from_dir, to_dir): return self._ftp_publish_command(from_dir + "/*", to_dir) def _ftp_publish_command(self, from_location, to_location): ssh_port = self.config.get('ftp', 'ssh_port') ssh_key = self.config.get('ftp', 'ssh_key') user = self.config.get('ftp', 'user') server = self.config.get('ftp', 'server') ssh_mkdir_command = ['ssh', "-i", ssh_key, '-p', ssh_port, user + '@' + server, '"mkdir ' + to_location + '"'] scp_command = ['scp', '-i', ssh_key, '-P', ssh_port, '-r', from_location, '"' + user + '@' + server + ':' + to_location +'"'] ssh_command = ['ssh', "-i", ssh_key, '-p', ssh_port, user + '@' + server, '"chmod -R g+r ' + to_location + ' && chown -R ' + user + ':www-data ' + to_location + '"'] # Flatten to a string so that a shell executes de command, and # expands ~ return ' '.join(ssh_mkdir_command) + ' ; ' + ' '.join(scp_command) + ' && ' + ' '.join(ssh_command) def make_wheel_builder(self): builder_name = "builder_wheels" venv_name = "virtualenv_wheels" factory = BuildFactory() generate_wheels = 'pkg/generate_wheels.sh' publish_wheels = self._publish_wheels_command() sandbox_path_top = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} sandbox_path_soledad = {'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} factory.addStep(ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv")) factory.addStep(ShellCommand(command=['pip', 'install', '-U', 'wheel'], env=sandbox_path_top, haltOnFailure=True, workdir=".", name="Install wheels")) for repo_name, git_branch, _, _ in self.repos: repo_url = github_repo_url(repo_name) workdir = repo_name factory.addStep( Git(repourl=repo_url, branch=git_branch, workdir=workdir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url)) if 'soledad' in repo_name: for subpackage in ["common", "client", "server"]: factory.addStep( ShellCommand(command=generate_wheels, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="wheels for " + repo_name+"."+subpackage)) else: factory.addStep( ShellCommand(command=generate_wheels, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="wheels for " + repo_name)) factory.addStep(ShellCommand(command=publish_wheels, env=sandbox_path, doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), workdir=".", name="publish wheels")) self._add_pyside_setup_repo(factory) return BuilderConfig(name=builder_name, slavenames=self.slaves.names(), factory=factory) def _publish_wheels_command(self): original_wheelhouse = self.config.get('ftp', 'copy_wheels_from') directory = self.config.get('ftp', 'directory') return self._ftp_publish_dir_command(original_wheelhouse, directory) def _add_pyside_setup_repo(self, factory): repo_name = "pyside-setup" repo_url = "https://github.com/ivanalejandro0/" + repo_name + ".git" git_branch = "master" venv_name = "virtualenv_wheels" sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} publish_pyside_wheel = self._publish_pyside_command('`ls -t *.whl | head -1`') factory.addSteps([ ShellCommand(command=['rm', '-rf', repo_name], workdir='.', env=sandbox_path, name="Remove previous pyside"), Git(repourl=repo_url, branch=git_branch, workdir=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), ShellCommand(command=['python', 'setup.py', 'bdist_wheel', '--standalone'], workdir=repo_name, env=sandbox_path, name="Wheel for " + repo_name), ShellCommand(command=publish_pyside_wheel, workdir=repo_name + '/dist/', name="Publish pyside") ]) def _publish_pyside_command(self, location): directory = self.config.get('ftp', 'directory') command = self._ftp_publish_command(location, directory) return command def make_bundler_builder(self): builder_name = "builder_bundler" factory = BuildFactory() repo_name = "bitmask_bundler" repo_url = "https://github.com/leapcode/" + repo_name + ".git" branch = "develop" workdir="build" repo_dir = workdir + "/" + repo_name bundler_output_dir = "bundler_output" sumo_tarball = "leap.bitmask-latest-SUMO.tar.gz" publish_bundle = self._publish_bundle_command('`ls -t *.tar.gz | head -1`') factory.addSteps([ Git(repourl=repo_url, branch=branch, workdir=repo_dir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), ShellCommand(command="rm -rf " + bundler_output_dir, workdir=workdir, name="Remove previous bundler dir"), ShellCommand(command="mkdir " + bundler_output_dir, workdir=workdir, name="Create bundler dir"), ShellCommand(command="cp bundle_pyinstaller.sh ../" + bundler_output_dir, workdir=repo_dir, haltOnFailure=True, name="Copy bundle_pyinstaller"), ShellCommand(command="mkdir files", workdir=workdir + '/' + bundler_output_dir, name="Create auxiliary folder"), ShellCommand(command="wget http://lizard.leap.se/sumo-tarball/" + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, haltOnFailure=True, name="Download sumo"), ShellCommand(command="./bundle_pyinstaller.sh " + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, name="Create bundle"), ShellCommand(command=publish_bundle, workdir=workdir + '/' + bundler_output_dir, name="Publish bundle") ]) return BuilderConfig(name=builder_name, slavenames=self.slaves.leap_names(), factory=factory) def _publish_bundle_command(self, location): directory = self.config.get('ftp', 'bundle_target_directory') command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'bitmask-latest.tar.gz') return command def _add_repo_to_factory(factory, repo_name, git_branch, namespace, venv_name): install_requirements = ( "pkg/pip_install_requirements.sh --use-leap-wheels") install_requirements_tests = """ if [ -f pkg/requirements-testing.pip ] then pkg/pip_install_requirements.sh --testing --use-leap-wheels fi """ install = "python setup.py develop" workdir = repo_name sandbox_path = {'PATH': "../" + venv_name + '/bin/' + ':${PATH}'} sandbox_path_soledad = { 'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} repo_url = github_repo_url(repo_name) factory.addStep( Git(repourl=repo_url, branch=git_branch, workdir=workdir, codebase=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url)) if 'bitmask_client' in repo_name: factory.addSteps([ ShellCommand(command='pkg/postmkvenv.sh', env=sandbox_path, haltOnFailure=False, workdir=workdir, name="postmkenv"), ShellCommand(command='make', env=sandbox_path, haltOnFailure=False, workdir=workdir, name="make") ]) if 'soledad.git' in repo_url: for subpackage in ["common", "client", "server"]: # keymanager doesn't need soledad.server if ('keymanager' in venv_name and subpackage is not "server" or 'keymanager' is not venv_name): factory.addSteps([ ShellCommand( command=install_requirements, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="reqs: " + repo_name+"."+subpackage), ShellCommand( command=install_requirements_tests, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="test reqs: " + repo_name+"."+subpackage), ShellCommand( command=install, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="Install " + repo_name+"."+subpackage) ]) else: factory.addSteps([ ShellCommand(command=install_requirements, env=sandbox_path, haltOnFailure=False, workdir=workdir, name="reqs: " + repo_name), ShellCommand(command=install_requirements_tests, env=sandbox_path, haltOnFailure=False, workdir=workdir, name="test reqs: " + repo_name), ShellCommand(command=install, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="Install " + repo_name) ])