# -*- python -*- # ex: set syntax=python: # This is a sample buildmaster config file. It must be installed as # 'master.cfg' in your buildmaster's base directory. # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. c = BuildmasterConfig = {} ####### BUILDSLAVES # The 'slaves' list defines the set of recognized buildslaves. Each element is # a BuildSlave object, specifying a unique slave name and password. The same # slave name and password must be configured on the slave. from buildbot.buildslave import BuildSlave # XXX get actual passwords ! c['slaves'] = [BuildSlave("linux-build-slave", "foobar"), # BuildSlave("windows-build-slave", "foobar"), # BuildSlave("mac-build-slave", "foobar") ] # 'slavePortnum' defines the TCP port to listen on for connections from slaves. # This must match the value configured into the buildslaves (with their # --master option) c['slavePortnum'] = 9989 ####### CHANGESOURCES # the 'change_source' setting tells the buildmaster how it should find out # about source code changes. import subprocess def run_cmd(cmd, cwd=None): p = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, cwd=cwd) stdout = p.communicate()[0] return stdout # to generate the list of remote branches we # 1. create bare repositry # 2. get the list of branches # 3. delete the repository #gitflow = [] #clone_cmd = "git clone --bare git://leap.se/leap_client tmp" #run_cmd(clone_cmd) #list_branch_cmd = "git branch --list feature/*" #stdout = run_cmd(list_branch_cmd, "tmp") #for line in stdout.split("\n"): # if len(line.strip())> 0: # gitflow.append(line.strip()) #rm_cmd = "rm -rf tmp" #run_cmd(rm_cmd) #branches = ['develop'] + gitflow from buildbot.changes.gitpoller import GitPoller from buildbot.changes import pb REPO_BASE = "https://leap.se/git/" # Tuples in the form: (repo_uri, repo_name) REPOS = (("bitmask_client", "bitmask"), ("soledad", "soledad"), ("leap_mail", "mail"), ("keymanager", "keymanager")) # This is only for having all builders having # a scheduler by now. Eventually, replace # REPOS with this one. REPOS_ALL = (("bitmask_client", "bitmask"), ("soledad", "soledad"), ("leap_mail", "mail"), ("keymanager", "keymanager"), ("leap_mx", "mx"), ("leap_pycommon", "common")) branches = ['develop', 'master'] c['change_source'] = [] for repo, proj_name in REPOS: c['change_source'].append(GitPoller( REPO_BASE + repo, workdir="gitpoller-workdir-"+repo, project=proj_name, branches=branches, pollinterval=300)) #git poll branch names to automatically build branches to watch. ####### SCHEDULERS # Configure the Schedulers, which decide how to react to incoming changes. In this # case, just kick off a 'runtests' build from buildbot.schedulers.basic import SingleBranchScheduler from buildbot.schedulers.forcesched import ForceScheduler from buildbot.schedulers.timed import Nightly from buildbot.changes import filter c['schedulers'] = [] # BRANCH SCHEDULER (each time a commit is done) # for develop and master branch, for all *active* # repos # --------------------------------------------- for _, proj_name in REPOS: for branch in branches: c['schedulers'].append( SingleBranchScheduler( name="%s-%-s-checkin" % (proj_name, branch), change_filter=filter.ChangeFilter( branch=branch, project=proj_name), treeStableTimer=60, builderNames=[ "%s-linux-quick" % proj_name, ])) # NIGHTLY SCHEDULER -- daily. # We should change to a new builder that # triggers an autobuild. # --------------------------------------------- for _, proj_name in REPOS_ALL: c['schedulers'].append(Nightly( name="%s-develop-nightly" % proj_name, branch="develop", hour=16, minute=30, onlyIfChanged=True, builderNames=["%s-linux-quick" % proj_name] )) ####### BUILDERS # The 'builders' list defines the Builders, which tell Buildbot how to perform a build: # what steps, and which slaves can execute them. Note that any particular build will # only take place on one slave. from buildbot.process.factory import BuildFactory from buildbot.steps.source.git import Git from buildbot.steps.shell import ShellCommand # XXX TODO lock down accepted commands class UploadCoverage(ShellCommand): def start(self): self.buildername = self.getProperty("buildername") self.branch = self.getProperty("branch") self.buildnumber = self.getProperty("buildnumber") cmd = """ssh -p 4422 buildmaster@lemur.leap.se mkdir -p buildmaster/public_html/coverage/%s/%s/ scp -P 4422 -r docs/covhtml/ buildmaster@lemur.leap.se:buildmaster/public_html/coverage/%s/%s/%s/""" % ( self.buildername, self.branch, self.buildername, self.branch, self.buildnumber) self.setCommand(cmd) ShellCommand.start(self) def createSummary(self, log): url = "http://lemur.leap.se:8010/coverage/%s/%s/%s/" % ( self.buildername, self.branch, self.buildnumber) self.addURL("coverage", url) # factories are responsible for creating new Build instances #cp_script = ShellCommand(name='Copy script', # description='copy script (need to push this XXX!)', # #command=['cp', 'pkg/scripts/install_venv.py', 'pkg/']) # command=['cp', '/home/buildslave/scripts/install_venv.py', 'pkg/']) #skip_pyside = ShellCommand(name='Skip pyside', # description='workaround for skipping pyside', # command=['sed', '-i', 's/PySide/#PySide/', 'pkg/requirements.pip']) #add_missing_dep = ShellCommand(name='Extra dep', # description='workaround for missing deps XXX -- push', # command=['sed', '-i', '$ a\oauth', 'pkg/requirements.pip']) #install_virt = ShellCommand(name='Installing Virtual Environment', # env={'LEAP_VENV_SKIP_PYSIDE': '1'}, # description='Installing Virtual Environment', # command=['python', 'pkg/install_venv.py']) # XXX Workaround for long installs --- should add another builder # that works inside a permanent virtualenv, using venvwrapper... install_virt = ShellCommand(name='Installing Virtual Environment', env={'LEAP_VENV_SKIP_PYSIDE': '1'}, description='Installing Virtual Environment', command=['cp', '-r', '/home/buildslave/venv/.venv', '.']) make_resources = ShellCommand(name='compile qt resources', description='make ui', command=['make']) run_tests = ShellCommand(name='run_tests', description='run_tests', command=['pkg/tools/with_venv.sh', './run_tests.sh', '--coverage']) develop_mode = ShellCommand(name='develop', description='setup develop', command=['pkg/tools/with_venvwrapper.sh', 'leap-bitmask', 'python', 'setup.py', 'develop']) develop_mode_soledad = ShellCommand(name='develop mode (soledad)', description='setup develop', command=['common/pkg/tools/with_venvwrapper.sh', 'leap-bitmask', './scripts/develop_mode.sh']) run_tests_venv = ShellCommand( name='tests (venv)', description="run_tests", command=['pkg/tools/with_venvwrapper.sh', 'leap-bitmask', 'xvfb-run', './run_tests.sh', '-N', '--coverage'], env={'HOME':'/home/buildslave', 'WORKON_HOME':'/home/buildslave/Virtualenvs', 'VIRTUALENVWRAPPER_HOOK_DIR':'/home/buildslave/Virtualenvs', 'VIRTUALENVWRAPPER_LOG_DIR':'/home/buildslave/Virtualenvs'}) # Why the fuck is the wrapper not working??? # 'PATH':'/home/buildslave/Virtualenvs/leap-bitmask/bin:/home/buildslave/Virtualenvs/leap-bitmask/local/lib/python2.7/site-packages:/usr/local/bin:/usr/bin:/bin'}) run_tests_py = ShellCommand( name='tests (venv)', description="run_tests", command=['pkg/tools/with_venvwrapper.sh', 'leap-bitmask', 'python', 'setup.py', 'test']) run_tests_py_soledad = ShellCommand( name='tests (venv)', description="run_tests", command=['common/pkg/tools/with_venvwrapper.sh', 'leap-bitmask', './run_tests.sh']) run_tox = ShellCommand(name='tox', description='tox', command=['pkg/tools/with_venv.sh', 'tox', '-v']) run_pep8 = ShellCommand(name='pep8', description='pep8', command=['pkg/tools/with_venvwrapper.sh', 'leap-bitmask', 'pep8', '--ignore=E202,W602 --exclude=ui_*,*_rc.py,_version.py --repeat', 'src/leap']) run_pep8_soledad = ShellCommand(name='pep8', description='pep8', command=['common/pkg/tools/with_venvwrapper.sh', 'leap-bitmask', 'pep8', '--ignore=E202,W602 --exclude=ui_*,*_rc.py,_version.py --repeat', 'common/src/leap']) run_coverage = ShellCommand(name='coverage', description='coverage', command=['pkg/tools/with_venv.sh', 'coverage', 'html', '-d', 'docs/covhtml', '-i', '--include=src/leap/*', '--omit=src/leap/base/tests/*,src/leap/eip/tests/*,src/leap/gui/tests/*,/src/leap/util/tests/*'], haltOnFailure=True) # Linux incremental ----------------------------------------------------------------- f1 = BuildFactory() f1.addStep(Git(repourl='git://leap.se/bitmask_client', mode='incremental', branch='develop')) f1.addStep(install_virt) f1.addStep(run_tests) f1.addStep(run_tox) f1.addStep(run_pep8) f1.addStep(run_coverage) f1.addStep(UploadCoverage(name='Uploading coverage', description='Uploading coverage')) # ------------------------------------------------------------------------------------- def get_quick_builder(repo_name): """ BUILDERS FOR LINUX QUICK """ client = mail = soledad = mx = False if repo_name == "bitmask_client": client = True name = "bitmask" elif repo_name == "leap_mail": mail = True name = "mail" elif repo_name == "leap_mx": mx = True name = "mx" elif repo_name == "soledad": soledad = True name = "soledad" elif repo_name == "leap_pycommon": name = "common" else: name = repo_name steps = [ Git(repourl='https://leap.se/git/%s' % repo_name, mode='full', branch='develop')] if client: steps.append(make_resources) if soledad: steps = steps + [ develop_mode_soledad] else: steps = steps + [ develop_mode] # run pep8 if soledad: steps = steps + [run_pep8_soledad] else: steps = steps + [run_pep8] # run tests if client: steps = steps + [ run_tests_venv, UploadCoverage(name='Uploading coverage', description='Uploading coverage')] elif soledad: steps = steps + [run_tests_py_soledad] else: # any other steps = steps + [run_tests_py] f = BuildFactory() for s in steps: f.addStep(s) b = {'name': "%s-linux-quick" % name, 'builddir': "%s_linux_quick" % name, 'slavename': "linux-build-slave", 'factory': f, } return b # multiple buildslaves in a builder provide redundancy # Difft platforms, OSes, versions, libraries should have dfft Builders b1 = {'name': "linux-incremental", 'slavename': "linux-build-slave", 'builddir': "linux-incremental", 'factory': f1, } repos = ['bitmask_client', 'leap_mail', 'soledad', 'keymanager', 'leap_pycommon', 'leap_mx'] c['builders'] = [get_quick_builder(repo) for repo in repos] ####### STATUS TARGETS # 'status' is a list of Status Targets. The results of each build will be # pushed to these targets. buildbot/status/*.py has a variety to choose from, # including web pages, email senders, and IRC bots. c['status'] = [] from buildbot.status import html from buildbot.status.web import authz, auth authz_cfg=authz.Authz( # change any of these to True to enable; see the manual for more # options auth=auth.BasicAuth([("foobar","foobar")]), gracefulShutdown = False, forceBuild = 'auth', # use this to test your slave once it is set up forceAllBuilds = False, pingBuilder = True, stopBuild = 'auth', stopAllBuilds = False, cancelPendingBuild = 'auth', ) c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) # Our dear polkabot! .... from buildbot.status import words # get this from the config file... BBOT_NICK = "polkabot" BBOT_CHANNELS = ["#borogoves"] c['status'].append(words.IRC( host="irc.indymedia.org", nick=BBOT_NICK, channels=BBOT_CHANNELS, allowForce=True, notify_events={ 'started': 1, 'finished': 1, 'failure': 1, } )) ####### PROJECT IDENTITY c['title'] = "LEAP" c['titleURL'] = "https://leap.se" c['buildbotURL'] = "http://lemur.leap.se:8010/" ####### DB URL c['db'] = { 'db_url' : "sqlite:///state.sqlite", }