Merge branch 'feature/7339-publish-wheels-and-sumo'

2 files changed, 86 insertions, 14 deletions

diff --git a/conf_template.cfg b/conf_template.cfg
index faed98e..4b59e89 100644
--- a/conf_template.cfg
+++ b/conf_template.cfg
@@ -1,3 +1,11 @@
 [Buildbot]
 url = url:port
 title = Buildbot
+
+[ftp]
+copy_wheels_from = ~/wheelhouse
+ssh_port  = port
+ssh_key = ~/.ssh/id
+user = user
+server = server
+directory = ~/wheels
\ No newline at end of file
diff --git a/master.cfg b/master.cfg
index 87e3d18..3d166aa 100644
--- a/master.cfg
+++ b/master.cfg
@@ -1,6 +1,12 @@
 # -*- python -*-
 # ex: set syntax=python:
 
+import textwrap
+import ConfigParser
+
+config = ConfigParser.ConfigParser()
+config.read('conf.cfg')
+
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}
@@ -91,7 +97,7 @@ def add_repo_to_factory(factory, repo_name, git_branch, namespace, venv_name):
     install_requirements_tests = "if [ -f pkg/requirements-testing.pip ]; then pkg/pip_install_requirements.sh --testing; fi"
     install = "python setup.py develop"
 
-    workdir = "workdir-" + repo_name
+    workdir = repo_name
     sandbox_path = {'PATH':  "../" + venv_name + '/bin/' + ':${PATH}'}
     sandbox_path_soledad = {'PATH':  "../../" + venv_name + '/bin/' + ':${PATH}'}
     repo_url = github_repo_url(repo_name)
@@ -133,26 +139,81 @@ def create_builder(repo_name):
         add_repo_to_factory(factory, repo_name, repo_branch, namespace, venv_name)
 
     factory.addStep(
-        ShellCommand(command=['pep8', '--exclude=setup.py,versioneer.py,src._version', '--ignore=E501', '.'],env=venv_path_factory,haltOnFailure=False, workdir="workdir-" + repo_name, name="pep8 on " + repo_name))
+        ShellCommand(command=['pep8', '--exclude=setup.py,versioneer.py,src._version', '--ignore=E501', '.'],env=venv_path_factory,haltOnFailure=False, workdir=repo_name, name="pep8 on " + repo_name))
 
     if namespace is not '':
-        factory.addStep(ShellCommand(command=['trial', namespace], env=venv_path_factory, workdir="workdir-" + repo_name, name="trial "+namespace))
+        factory.addStep(ShellCommand(command=['trial', namespace], env=venv_path_factory, workdir=repo_name, name="trial "+namespace))
 
     if repo_name is 'bitmask_client':
-        factory.addStep(ShellCommand(command=['make', 'sumo_tarball'],
-                                     env=venv_path_factory, workdir="workdir-" + repo_name,
-                                     doStepIf=(lambda step: step.getProperty('slavename') == localhost_slave),
-                                     name="make sumo tarball"))
+        publish_sumo = publish_sumo_command('`ls -t *SUMO.tar.gz | head -1`')
+        factory.addSteps(
+            [ShellCommand(command=['make', 'sumo_tarball'],
+                          env=venv_path_factory, workdir=repo_name,
+                          doStepIf=(lambda step: step.getProperty('slavename') == localhost_slave),
+                          name="make sumo tarball"),
+             ShellCommand(command=publish_sumo,
+                          env=venv_path_factory, workdir=repo_name + "/dist",
+                          doStepIf=(lambda step: step.getProperty('slavename') == localhost_slave),
+                          name="publish sumo to ftp")
+            ])
 
 
     return BuilderConfig(name=builder_name, slavenames=[localhost_slave], factory=factory)
 
+def publish_sumo_command(location):
+    directory = config.get('ftp', 'sumo_target_directory')
+    command = ftp_publish_command(location, directory) + '&&' + ftp_soft_link_sumo(location)
+    
+    return command
+
+def ftp_soft_link_sumo(filename):
+    directory = config.get('ftp', 'sumo_target_directory')
+    ssh_port = config.get('ftp', 'ssh_port')
+    ssh_key = config.get('ftp', 'ssh_key')
+    user = config.get('ftp', 'user')
+    server = config.get('ftp', 'server')
+
+    ssh_command = ['ssh',
+                   "-i", ssh_key,
+                   '-p', ssh_port,
+                   user + '@' + server,
+                   '"ln -sf ' + directory + '/' + filename + ' ' + directory + '/leap.bitmask-latest-SUMO.tar.gz"']
+
+    # Flatten to a string so that a shell executes de command, and
+    # expands ~
+    return ' '.join(ssh_command)
+
+def ftp_publish_dir_command(from_dir, to_dir):
+    return ftp_publish_command(from_dir + "/*", to_dir)
+
+def ftp_publish_command(from_location, to_location):
+    ssh_port = config.get('ftp', 'ssh_port')
+    ssh_key = config.get('ftp', 'ssh_key')
+    user = config.get('ftp', 'user')
+    server = config.get('ftp', 'server')
+
+    scp_command = ['scp',
+                   '-i', ssh_key,
+                   '-P', ssh_port,
+                   '-r', from_location,
+                   user + '@' + server + ':' + to_location]
+    ssh_command = ['ssh',
+                   "-i", ssh_key,
+                   '-p', ssh_port,
+                   user + '@' + server,
+                   '"chmod g+r ' + to_location + ' && chown -R ' + user + ':www-data ' + to_location + '"']
+    # Flatten to a string so that a shell executes de command, and
+    # expands ~
+    return ' '.join(scp_command) + ' && ' + ' '.join(ssh_command)
+
 def make_wheel_builder():
     builder_name = "builder_wheels"
     venv_name = "virtualenv_wheels"
     factory = BuildFactory()
 
     generate_wheels = 'pkg/generate_wheels.sh'
+    publish_wheels = publish_wheels_command()
+
     sandbox_path_top = {'PATH':  "./" + venv_name + '/bin' + ':${PATH}'}
     sandbox_path = {'PATH':  "../" + venv_name + '/bin' + ':${PATH}'}
     sandbox_path_soledad = {'PATH':  "../../" + venv_name + '/bin/' + ':${PATH}'}
@@ -161,19 +222,26 @@ def make_wheel_builder():
     factory.addStep(ShellCommand(command=['pip', 'install', '-U', 'wheel'], env=sandbox_path_top, haltOnFailure=True, workdir=".", name="Install wheels"))
     for repo_name, git_branch, _, _ in REPOS:
         repo_url = github_repo_url(repo_name)
-        workdir = "workdir-"+repo_name
+        workdir = repo_name
         factory.addStep(
             Git(repourl=repo_url, branch=git_branch, workdir=workdir, mode='incremental', method='clean', haltOnFailure=True, name="Pull " + repo_url))
         if 'soledad' in repo_name:
             for subpackage in ["common", "client", "server"]:
                 factory.addStep(
-                    ShellCommand(command=generate_wheels, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="reqs: " + repo_name+"."+subpackage))
+                    ShellCommand(command=generate_wheels, env=sandbox_path_soledad, haltOnFailure=True, workdir=workdir+'/'+subpackage, name="wheels for " + repo_name+"."+subpackage))
         else:
             factory.addStep(
-                ShellCommand(command=generate_wheels, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="reqs: " + repo_name))
+                ShellCommand(command=generate_wheels, env=sandbox_path, haltOnFailure=True, workdir=workdir, name="wheels for " + repo_name))
+    factory.addStep(ShellCommand(command=publish_wheels, env=sandbox_path, haltOnFailure=True, workdir=".", name="publish wheels"))
 
     return BuilderConfig(name=builder_name, slavenames=[localhost_slave], factory=factory)
 
+def publish_wheels_command():
+    original_wheelhouse = config.get('ftp', 'copy_wheels_from')
+    directory = config.get('ftp', 'directory')
+
+    return ftp_publish_dir_command(original_wheelhouse, directory)
+
 c['builders'] = []
 
 for repo_name, _, _, _ in REPOS:
@@ -195,10 +263,6 @@ c['www'] = dict(port=PORT_WEB,
 
 ####### PROJECT IDENTITY
 
-import ConfigParser
-config = ConfigParser.ConfigParser()
-config.read('conf.cfg')
-
 # the 'title' string will appear at the top of this buildbot
 # installation's html.WebStatus home page (linked to the
 # 'titleURL') and is embedded in the title of the waterfall HTML page.