diff options
Diffstat (limited to 'builders.py')
-rw-r--r-- | builders.py | 321 |
1 files changed, 321 insertions, 0 deletions
diff --git a/builders.py b/builders.py new file mode 100644 index 0000000..35cd9b3 --- /dev/null +++ b/builders.py @@ -0,0 +1,321 @@ +from util import github_repo_url, order_repos_index + +from buildbot.process.factory import BuildFactory +from buildbot.steps.source.git import Git +from buildbot.steps.shell import ShellCommand +from buildbot.config import BuilderConfig + + +class Builders(object): + + def __init__(self, config, slaves, repos): + self.config = config + self.slaves = slaves + self.repos = repos + + def create_builder(self, repo_name): + builder_name = 'builder_' + repo_name + venv_name = "virtualenv_ci_" + builder_name + venv_path = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} + venv_path_factory = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} + + factory = BuildFactory() + factory.addSteps([ + ShellCommand(command=["rm", "-rf", venv_name], haltOnFailure=True, + workdir=".", name="Remove previous virtualenv"), + ShellCommand(command=["virtualenv", "--python=python2", venv_name], + haltOnFailure=True, workdir=".", + name="Create new virtualenv"), + ShellCommand( + command=['pip', 'install', '-U', 'pip', 'setuptools', + 'coverage'], + env=venv_path, workdir=".", name="Update setuptools") + ]) + + repo_index = [repo[order_repos_index] for repo in self.repos if repo[0] is repo_name][0] + for repo_name, repo_branch, _, namespace, in sorted(self.repos, key = lambda repo: repo[order_repos_index])[0:repo_index]: + _add_repo_to_factory(factory, repo_name, repo_branch, namespace, venv_name) + + factory.addSteps([ + ShellCommand(command=['pep8', '.'],env=venv_path_factory,haltOnFailure=False, workdir=repo_name, name="pep8 on " + repo_name)]) + + if namespace is not '': + if repo_name is 'bitmask_client': + factory.addStep( + ShellCommand(command=['xvfb-run', 'coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) + else: + factory.addStep( + ShellCommand(command=['coverage', 'run', '--omit=*/'+venv_name+'/*', venv_name + '/bin/trial', namespace], env=venv_path, workdir='.', name="trial "+namespace)) + + factory.addSteps([ + ShellCommand(command=['coverage', 'html'], env=venv_path, workdir='.', name="generate html coverage report for " +namespace), + ShellCommand(command=self._publish_coverage_reports_command('htmlcov', repo_name), workdir='.', doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename')))) + ]) + + self._publish_leap_wheels(factory, repo_name, venv_path_factory, doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename')))) + + if repo_name == 'bitmask_client': + publish_sumo = self._publish_sumo_command('`ls -t *SUMO.tar.gz | head -1`') + + factory.addSteps([ + ShellCommand(command=['make', 'sumo_tarball_latest'], + env=venv_path_factory, workdir=repo_name, + doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), + name="make sumo tarball"), + ShellCommand(command=publish_sumo, + env=venv_path_factory, workdir=repo_name + "/dist", + doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), + name="publish sumo to ftp") + ]) + + + return BuilderConfig(name=builder_name, slavenames=self.slaves.names(), factory=factory) + + def _publish_coverage_reports_command(self, location, repo_name): + target_directory = self.config.get('ftp', 'coverage_reports_target_directory') + '/' + repo_name + '_' + '`git -C ' + repo_name + ' describe`' + + return self._ftp_publish_dir_command(location, target_directory) + + def _publish_leap_wheels(self, factory, repo_name, env, doStepIf): + env_soledad = {'PATH': env['PATH'].replace('../', '../../', 1)} + + if repo_name == 'soledad': + for subpackage in ["common", "client", "server"]: + factory.addSteps([ + ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage, name="leap wheels for " + repo_name+"."+subpackage), + ShellCommand(command=self._publish_leap_wheels_soledad(subpackage, '`ls -t *.whl | head -1`'), env=env_soledad, doStepIf=doStepIf, haltOnFailure=True, workdir=repo_name+'/'+subpackage+'/dist', name="publish leap wheels for " + repo_name+"."+subpackage)]) + else: + factory.addSteps([ + ShellCommand(command=['python', 'setup.py', 'bdist_wheel'], env=env, doStepIf=doStepIf, workdir=repo_name, name="Generate leap wheels for "+repo_name), + ShellCommand(command=self._publish_leap_wheels_command(repo_name, '`ls -t *.whl | head -1`'), env=env, doStepIf=doStepIf, workdir=repo_name + '/dist', name="Publish leap wheels for "+repo_name) + ]) + + def _publish_leap_wheels_command(self, repo_name, location): + directory = self.config.get('ftp', 'leap_wheels_directory') + command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.' + repo_name + '-latest.whl') + + return command + + def _publish_leap_wheels_soledad(self, subpackage, location): + directory = self.config.get('ftp', 'leap_wheels_directory') + command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.soledad.' + subpackage + '-latest.whl') + + return command + + def _publish_sumo_command(self, location): + directory = self.config.get('ftp', 'sumo_target_directory') + command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'leap.bitmask-latest-SUMO.tar.gz') + + return command + + def _ftp_soft_link(self, filename, target_directory, symlink_name): + return self._ftp_ssh_command('ln -sf ' + target_directory + '/' + filename + ' ' + target_directory + '/' + symlink_name) + + def _ftp_ssh_command(self, command): + ssh_port = self.config.get('ftp', 'ssh_port') + ssh_key = self.config.get('ftp', 'ssh_key') + user = self.config.get('ftp', 'user') + server = self.config.get('ftp', 'server') + + ssh_command = ['ssh', + "-i", ssh_key, + '-p', ssh_port, + user + '@' + server, + '"' + command + '"'] + + # Flatten to a string so that a shell executes de command, and + # expands ~ + return ' '.join(ssh_command) + + def _ftp_publish_dir_command(self, from_dir, to_dir): + return self._ftp_publish_command(from_dir + "/*", to_dir) + + def _ftp_publish_command(self, from_location, to_location): + ssh_port = self.config.get('ftp', 'ssh_port') + ssh_key = self.config.get('ftp', 'ssh_key') + user = self.config.get('ftp', 'user') + server = self.config.get('ftp', 'server') + + ssh_mkdir_command = ['ssh', + "-i", ssh_key, + '-p', ssh_port, + user + '@' + server, + '"mkdir ' + to_location + '"'] + + scp_command = ['scp', + '-i', ssh_key, + '-P', ssh_port, + '-r', from_location, + '"' + user + '@' + server + ':' + to_location +'"'] + ssh_command = ['ssh', + "-i", ssh_key, + '-p', ssh_port, + user + '@' + server, + '"chmod -R g+r ' + to_location + ' && chown -R ' + user + ':www-data ' + to_location + '"'] + # Flatten to a string so that a shell executes de command, and + # expands ~ + return ' '.join(ssh_mkdir_command) + ' ; ' + ' '.join(scp_command) + ' && ' + ' '.join(ssh_command) + + def make_wheel_builder(self): + builder_name = "builder_wheels" + venv_name = "virtualenv_wheels" + factory = BuildFactory() + + generate_wheels = 'pkg/generate_wheels.sh' + publish_wheels = self._publish_wheels_command() + + sandbox_path_top = {'PATH': "./" + venv_name + '/bin' + ':${PATH}'} + sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} + sandbox_path_soledad = {'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} + + factory.addStep(ShellCommand(command=["virtualenv", "--python=python2", venv_name], haltOnFailure=True, workdir=".", name="Create new virtualenv")) + factory.addStep(ShellCommand(command=['pip', 'install', '-U', 'wheel'], env=sandbox_path_top, haltOnFailure=True, workdir=".", name="Install wheels")) + for repo_name, git_branch, _, _ in self.repos: + repo_url = github_repo_url(repo_name) + workdir = repo_name + factory.addStep( + Git(repourl=repo_url, branch=git_branch, workdir=workdir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url)) + if 'soledad' in repo_name: + for subpackage in ["common", "client", "server"]: + factory.addStep( + ShellCommand(command=generate_wheels, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="wheels for " + repo_name+"."+subpackage)) + else: + factory.addStep( + ShellCommand(command=generate_wheels, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="wheels for " + repo_name)) + factory.addStep(ShellCommand(command=publish_wheels, env=sandbox_path, doStepIf=(lambda step: self.slaves.is_leap(step.getProperty('slavename'))), workdir=".", name="publish wheels")) + + self._add_pyside_setup_repo(factory) + + return BuilderConfig(name=builder_name, slavenames=self.slaves.names(), factory=factory) + + def _publish_wheels_command(self): + original_wheelhouse = self.config.get('ftp', 'copy_wheels_from') + directory = self.config.get('ftp', 'directory') + + return self._ftp_publish_dir_command(original_wheelhouse, directory) + + def _add_pyside_setup_repo(self, factory): + repo_name = "pyside-setup" + repo_url = "https://github.com/ivanalejandro0/" + repo_name + ".git" + git_branch = "master" + + venv_name = "virtualenv_wheels" + sandbox_path = {'PATH': "../" + venv_name + '/bin' + ':${PATH}'} + + publish_pyside_wheel = self._publish_pyside_command('`ls -t *.whl | head -1`') + factory.addSteps([ + ShellCommand(command=['rm', '-rf', repo_name], workdir='.', env=sandbox_path, name="Remove previous pyside"), + Git(repourl=repo_url, branch=git_branch, workdir=repo_name, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), + ShellCommand(command=['python', 'setup.py', 'bdist_wheel', '--standalone'], workdir=repo_name, env=sandbox_path, name="Wheel for " + repo_name), + ShellCommand(command=publish_pyside_wheel, workdir=repo_name + '/dist/', name="Publish pyside") + ]) + + def _publish_pyside_command(self, location): + directory = self.config.get('ftp', 'directory') + command = self._ftp_publish_command(location, directory) + + return command + + def make_bundler_builder(self): + builder_name = "builder_bundler" + factory = BuildFactory() + repo_name = "bitmask_bundler" + repo_url = "https://github.com/leapcode/" + repo_name + ".git" + branch = "develop" + + workdir="build" + repo_dir = workdir + "/" + repo_name + bundler_output_dir = "bundler_output" + sumo_tarball = "leap.bitmask-latest-SUMO.tar.gz" + + publish_bundle = self._publish_bundle_command('`ls -t *.tar.gz | head -1`') + + factory.addSteps([ + Git(repourl=repo_url, branch=branch, workdir=repo_dir, mode='full', method='clobber', shallow=True, haltOnFailure=True, name="Pull " + repo_url), + ShellCommand(command="rm -rf " + bundler_output_dir, workdir=workdir, name="Remove previous bundler dir"), + ShellCommand(command="mkdir " + bundler_output_dir, workdir=workdir, name="Create bundler dir"), + ShellCommand(command="cp bundle_pyinstaller.sh ../" + bundler_output_dir, workdir=repo_dir, haltOnFailure=True, name="Copy bundle_pyinstaller"), + ShellCommand(command="mkdir files", workdir=workdir + '/' + bundler_output_dir, name="Create auxiliary folder"), + ShellCommand(command="wget http://lizard.leap.se/sumo-tarball/" + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, haltOnFailure=True, name="Download sumo"), + ShellCommand(command="./bundle_pyinstaller.sh " + sumo_tarball, workdir=workdir + '/' + bundler_output_dir, name="Create bundle"), + ShellCommand(command=publish_bundle, workdir=workdir + '/' + bundler_output_dir, name="Publish bundle") + ]) + + return BuilderConfig(name=builder_name, slavenames=self.slaves.leap_names(), factory=factory) + + def _publish_bundle_command(self, location): + directory = self.config.get('ftp', 'bundle_target_directory') + command = self._ftp_publish_command(location, directory) + ' && ' + self._ftp_soft_link(location, directory, 'bitmask-latest.tar.gz') + + return command + + +def _add_repo_to_factory(factory, repo_name, git_branch, namespace, venv_name): + install_requirements = ( + "pkg/pip_install_requirements.sh --use-leap-wheels") + install_requirements_tests = """ + if [ -f pkg/requirements-testing.pip ] + then + pkg/pip_install_requirements.sh --testing --use-leap-wheels + fi + """ + install = "python setup.py develop" + + workdir = repo_name + sandbox_path = {'PATH': "../" + venv_name + '/bin/' + ':${PATH}'} + sandbox_path_soledad = { + 'PATH': "../../" + venv_name + '/bin/' + ':${PATH}'} + repo_url = github_repo_url(repo_name) + + factory.addStep( + Git(repourl=repo_url, branch=git_branch, workdir=workdir, + codebase=repo_name, mode='full', method='clobber', + shallow=True, haltOnFailure=True, name="Pull " + repo_url)) + + if 'bitmask_client' in repo_name: + factory.addSteps([ + ShellCommand(command='pkg/postmkvenv.sh', env=sandbox_path, + haltOnFailure=False, workdir=workdir, + name="postmkenv"), + ShellCommand(command='make', env=sandbox_path, + haltOnFailure=False, workdir=workdir, + name="make") + ]) + if 'soledad.git' in repo_url: + for subpackage in ["common", "client", "server"]: + # keymanager doesn't need soledad.server + if ('keymanager' in venv_name and + subpackage is not "server" or + 'keymanager' is not venv_name): + factory.addSteps([ + ShellCommand( + command=install_requirements, + env=sandbox_path_soledad, + haltOnFailure=True, + workdir=workdir+'/'+subpackage, + name="reqs: " + repo_name+"."+subpackage), + ShellCommand( + command=install_requirements_tests, + env=sandbox_path_soledad, + haltOnFailure=True, + workdir=workdir+'/'+subpackage, + name="test reqs: " + repo_name+"."+subpackage), + ShellCommand( + command=install, + env=sandbox_path_soledad, + haltOnFailure=True, + workdir=workdir+'/'+subpackage, + name="Install " + repo_name+"."+subpackage) + ]) + else: + factory.addSteps([ + ShellCommand(command=install_requirements, env=sandbox_path, + haltOnFailure=False, workdir=workdir, + name="reqs: " + repo_name), + ShellCommand(command=install_requirements_tests, + env=sandbox_path, haltOnFailure=False, + workdir=workdir, name="test reqs: " + repo_name), + ShellCommand(command=install, env=sandbox_path, + haltOnFailure=True, workdir=workdir, + name="Install " + repo_name) + ]) |