From 870cc8d8d51c600d1b9faa0a2d6a5a9dba8b5354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Thu, 21 Nov 2013 12:31:23 -0300 Subject: Create bundler for OSX --- bundler/actions.py | 443 ++++++++++++++++++++++++++++++++++++++++++++++ bundler/create_paths.py | 18 ++ bundler/darwin_dyliber.py | 59 ++++++ bundler/depcollector.py | 112 ++++++++++++ bundler/main.py | 116 ++++++++++++ bundler/utils.py | 3 + 6 files changed, 751 insertions(+) create mode 100644 bundler/actions.py create mode 100644 bundler/create_paths.py create mode 100644 bundler/darwin_dyliber.py create mode 100644 bundler/depcollector.py create mode 100644 bundler/main.py create mode 100644 bundler/utils.py (limited to 'bundler') diff --git a/bundler/actions.py b/bundler/actions.py new file mode 100644 index 0000000..6b977d4 --- /dev/null +++ b/bundler/actions.py @@ -0,0 +1,443 @@ +import os +import stat +import sys + +from abc import ABCMeta, abstractmethod +from contextlib import contextmanager +from distutils import file_util, dir_util + +from sh import git, cd, python, mkdir, make, cp, glob, pip, rm +from sh import find, SetFile, hdiutil, ln + +from utils import IS_MAC +from depcollector import collect_deps +from darwin_dyliber import fix_all_dylibs + +class Action(object): + __metaclass__ = ABCMeta + + def __init__(self, name, basedir, skip=[], do=[]): + self._name = name + self._basedir = basedir + self._skip = skip + self._do = do + + @property + def name(self): + return self._name + + @property + def skip(self): + return self._name in self._skip + + @property + def do(self): + if len(self._do) > 0: + return self._name in self._do + return True + + @abstractmethod + def run(self, *args, **kwargs): + pass + +def skippable(func): + def skip_func(self, *args, **kwargs): + if self.skip: + print "Skipping...", self.name + return + if not self.do: + print "Skipping...", self.name + return + return func(self, *args, **kwargs) + return skip_func + +def platform_dir(basedir, *args): + dir = os.path.join(basedir, + "Bitmask", + *args) + if IS_MAC: + dir = os.path.join(basedir, + "Bitmask", + "Bitmask.app", + "Contents", + "MacOS", + *args) + return dir + +@contextmanager +def push_pop(*directories): + cd(os.path.join(*directories)) + yield + cd(os.path.join(*(("..",)*len(directories)))) + +class GitCloneAll(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "gitclone", basedir, skip, do) + + def _repo_url(self, repo_name): + if repo_name == "leap_assets": + return "git://leap.se/leap_assets" + return "https://github.com/leapcode/{0}".format(repo_name) + + @skippable + def run(self, sorted_repos, nightly): + print "Cloning repositories..." + cd(self._basedir) + for repo in sorted_repos: + print "Cloning", repo + rm("-rf", repo) + git.clone(self._repo_url(repo), repo) + with push_pop(repo): + # Thandy is a special case regarding branches, we'll just use + # develop + if repo in ["thandy", "leap_assets"]: + continue + if not nightly: + git.checkout("master") + git.pull("--ff-only", "origin", "master") + git.fetch() + git.reset("--hard", "origin/master") + latest_tag = git.describe("--abbrev=0").strip() + git.checkout("--quiet", latest_tag) + else: + git.checkout("develop") + + print "Done cloning repos..." + +class PythonSetupAll(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "pythonsetup", basedir, skip, do) + + @skippable + def run(self, sorted_repos): + cd(self._basedir) + for repo in sorted_repos: + print "Setting up", repo + if repo == "soledad": + for subrepo in ["common", "client"]: + with push_pop(repo, subrepo): + pip("install", "-r", "pkg/requirements.pip") + python("setup.py", "develop") + sys.path.append(os.path.join(self._basedir, repo, subrepo, "src")) + elif repo in ["bitmask_launcher", "leap_assets"]: + print "Skipping launcher..." + continue + else: + with push_pop(repo): + if repo != "thandy": + pip("install", "-r", "pkg/requirements.pip") + else: + # Thandy is a special kid at this point in + # terms of packaging. So we install + # dependencies ourselves for the time being + pip("install", "pycrypto") + if repo == "bitmask_client": + print "Running make on the client..." + make() + print "Running build to get correct version..." + python("setup.py", "build") + python("setup.py", "develop") + sys.path.append(os.path.join(self._basedir, repo, "src")) + +class CreateDirStructure(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "createdirs", basedir, skip, do) + + @skippable + def run(self): + print "Creating directory structure..." + if IS_MAC: + self._darwin_create_dir_structure() + self._create_dir_structure(os.path.join(self._basedir, "Bitmask.app", "Contents", "MacOS")) + else: + self._create_dir_structure(self._basedir) + print "Done" + + def _create_dir_structure(self, basedir): + mkdirp = mkdir.bake("-p") + apps = os.path.join(basedir, "apps") + mkdirp(apps) + if not IS_MAC: + mkdirp(os.path.join(apps, "eip", "files")) + mkdirp(os.path.join(apps, "mail")) + mkdirp(os.path.join(basedir, "lib")) + + def _darwin_create_dir_structure(self): + mkdirp = mkdir.bake("-p") + app_path = os.path.join(self._basedir, "Bitmask.app") + mkdirp(app_path) + mkdirp(os.path.join(app_path, "Contents", "MacOS")) + mkdirp(os.path.join(app_path, "Contents", "Resources")) + mkdirp(os.path.join(app_path, "Contents", "PlugIns")) + mkdirp(os.path.join(app_path, "Contents", "StartupItems")) + ln("-s", "/Applications", os.path.join(self._basedir, "Applications")) + +class CollectAllDeps(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "collectdeps", basedir, skip, do) + + def _remove_unneeded(self, lib_dir): + print "Removing unneeded files..." + files = find(lib_dir).strip().splitlines() + for f in files: + if f.find("PySide") > 0: + if os.path.split(f)[1] not in ["QtCore.so", + "QtGui.so", + "__init__.py", + "_utils.py", + "PySide", + ""]: # empty means the whole pyside dir + rm("-rf", f) + print "Done" + + @skippable + def run(self, path_file): + print "Collecting dependencies..." + app_py = os.path.join(self._basedir, + "bitmask_client", + "src", + "leap", + "bitmask", + "app.py") + dest_lib_dir = platform_dir(self._basedir, "lib") + collect_deps(app_py, dest_lib_dir, path_file) + + self._remove_unneeded(dest_lib_dir) + print "Done" + +class CopyBinaries(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "copybinaries", basedir, skip, do) + + @skippable + def run(self, binaries_path): + print "Copying binaries..." + dest_lib_dir = platform_dir(self._basedir, "lib") + cp(glob(os.path.join(binaries_path, "Qt*")), dest_lib_dir) + cp(glob(os.path.join(binaries_path, "*.dylib")), dest_lib_dir) + cp(glob(os.path.join(binaries_path, "Python")), dest_lib_dir) + + if IS_MAC: + resources_dir = os.path.join(self._basedir, + "Bitmask", + "Bitmask.app", + "Contents", + "Resources") + cp(glob(os.path.join(binaries_path, "openvpn.leap*")), resources_dir) + + mkdir("-p", os.path.join(resources_dir, "openvpn")) + cp("-r", glob(os.path.join(binaries_path, "openvpn.files", "*")), os.path.join(resources_dir, "openvpn")) + + cp(os.path.join(binaries_path, "cocoasudo"), resources_dir) + + cp("-r", os.path.join(binaries_path, "qt_menu.nib"), resources_dir) + cp("-r", os.path.join(binaries_path, "tuntap-installer.app"), resources_dir) + else: + eip_dir = platform_dir(self._basedir, "apps", "eip") + cp(glob(os.path.join(binaries_path, "openvpn.leap*")), eip_dir) + + mkdir(os.path.join(resources_dir, "openvpn")) + cp("-r", glob(os.path.join(binaries_path, "openvpn.files", "*")), os.path.join(eip_dir, "files")) + + mail_dir = platform_dir(self._basedir, "apps", "mail") + cp(os.path.join(binaries_path, "gpg"), mail_dir) + cp(os.path.join(binaries_path, "Bitmask"), platform_dir(self._basedir)) + print "Done" + +class PLister(Action): + plist = """ + + + + CFBundleDisplayName + Bitmask + CFBundleExecutable + MacOS/bitmask-launcher + CFBundleIconFile + bitmask.icns + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Bitmask + CFBundlePackageType + APPL + CFBundleShortVersionString + 1 + LSBackgroundOnly + + +""".split("\n") + + qtconf = """[Paths] +Plugins = PlugIns""" + + def __init__(self, basedir, skip, do): + Action.__init__(self, "plister", basedir, skip, do) + + @skippable + def run(self): + print "Generating Info.plist file..." + file_util.write_file(os.path.join(self._basedir, + "Bitmask", + "Bitmask.app", + "Contents", + "Info.plist"), + self.plist) + print "Generating qt.conf file..." + file_util.write_file(os.path.join(self._basedir, + "Bitmask", + "Bitmask.app", + "Contents", + "Resources", + "qt.conf"), + self.qtconf) + print "Done" + +class SeededConfig(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "seededconfig", basedir, skip, do) + + @skippable + def run(self, seeded_config): + print "Copying seeded config..." + dir_util.copy_tree(seeded_config, + platform_dir(self._basedir, "config")) + print "Done" + +class DarwinLauncher(Action): + launcher = """#!/bin/bash +# +# Launcher for the LEAP Client under OSX +# +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" +export DYLD_LIBRARY_PATH=$DIR/lib +export PATH=$DIR/../Resources/:$PATH +# --------------------------- +# DEBUG Info -- enable this if you +# are having problems with dynamic libraries loading + +cd "${DIR}" && ./Bitmask $1 $2 $3 $4 $5""".split("\n") + + def __init__(self, basedir, skip, do): + Action.__init__(self, "darwinlauncher", basedir, skip, do) + + @skippable + def run(self): + print "Generating launcher script for OSX..." + launcher_path = os.path.join(self._basedir, + "Bitmask", + "Bitmask.app", + "Contents", + "MacOS", + "bitmask-launcher") + file_util.write_file(launcher_path, self.launcher) + os.chmod(launcher_path, stat.S_IRGRP | stat.S_IROTH | stat.S_IRUSR \ + | stat.S_IWGRP | stat.S_IWOTH | stat.S_IWUSR \ + | stat.S_IXGRP | stat.S_IXOTH | stat.S_IXUSR) + print "Done" + +class CopyAssets(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "copyassets", basedir, skip, do) + + @skippable + def run(self): + print "Copying assets..." + resources_dir = os.path.join(self._basedir, + "Bitmask", + "Bitmask.app", + "Contents", + "Resources") + cp(os.path.join(self._basedir, "leap_assets", "mac", "bitmask.icns"), + resources_dir) + cp(os.path.join(self._basedir, "leap_assets", "mac", "leap-client.tiff"), + resources_dir) + print "Done" + +class CopyMisc(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "copymisc", basedir, skip, do) + + @skippable + def run(self): + print "Copying misc files..." + apps_dir = platform_dir(self._basedir, "apps") + cp(os.path.join(self._basedir, "bitmask_launcher", "src", "launcher.py"), + apps_dir) + cp("-r", os.path.join(self._basedir, "thandy", "lib", "thandy"), + apps_dir) + cp("-r", os.path.join(self._basedir, "bitmask_client", "src", "leap"), + apps_dir) + lib_dir = platform_dir(self._basedir, "lib") + cp(os.path.join(self._basedir, + "leap_pycommon", + "src", "leap", "common", "cacert.pem"), + os.path.join(lib_dir, "leap", "common")) + cp(os.path.join(self._basedir, + "bitmask_client", "build", + "lib", "leap", "bitmask", "_version.py"), + os.path.join(apps_dir, "leap", "bitmask")) + + cp(os.path.join(self._basedir, + "bitmask_client", "relnotes.txt"), + os.path.join(self._basedir, "Bitmask")) + print "Done" + +class FixDylibs(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "fixdylibs", basedir, skip, do) + + @skippable + def run(self): + fix_all_dylibs(platform_dir(self._basedir)) + +class DmgIt(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "dmgit", basedir, skip, do) + + @skippable + def run(self): + cd(self._basedir) + version = "unknown" + with push_pop("bitmask_client"): + version = git("describe").strip() + dmg_dir = os.path.join(self._basedir, "dmg") + template_dir = os.path.join(self._basedir, "Bitmask") + mkdir("-p", dmg_dir) + cp("-R", os.path.join(template_dir, "Applications"), dmg_dir) + cp("-R", os.path.join(template_dir, "relnotes.txt"), dmg_dir) + cp("-R", os.path.join(template_dir, "Bitmask.app"), dmg_dir) + cp(os.path.join(self._basedir, + "leap_assets", + "mac", "bitmask.icns"), + os.path.join(dmg_dir, ".VolumeIcon.icns")) + SetFile("-c", "icnC", os.path.join(dmg_dir, ".VolumeIcon.icns")) + + vol_name = "Bitmask" + dmg_name = "Bitmask-OSX-{0}.dmg".format(version) + raw_dmg_path = os.path.join(self._basedir, "raw-{0}".format(dmg_name)) + dmg_path = os.path.join(self._basedir, dmg_name) + + hdiutil("create", "-srcfolder", dmg_dir, "-volname", vol_name, + "-format", "UDRW", "-ov", + raw_dmg_path) + rm("-rf", dmg_dir) + mkdir(dmg_dir) + hdiutil("attach", raw_dmg_path, "-mountpoint", dmg_dir) + SetFile("-a", "C", dmg_dir) + hdiutil("detach", dmg_dir) + + rm("-rf", dmg_dir) + hdiutil("convert", raw_dmg_path, "-format", "UDZO", "-o", + dmg_path) + rm("-f", raw_dmg_path) + +class PycRemover(Action): + def __init__(self, basedir, skip, do): + Action.__init__(self, "removepyc", basedir, skip, do) + + @skippable + def run(self): + print "Removing .pyc files..." + find(self._basedir, "-name", "\"*.pyc\"", "-delete") + print "Done" diff --git a/bundler/create_paths.py b/bundler/create_paths.py new file mode 100644 index 0000000..137e6e6 --- /dev/null +++ b/bundler/create_paths.py @@ -0,0 +1,18 @@ +import sys +from distutils import file_util + +def main(): + if len(sys.argv) != 2: + print "ERROR: Wrong amount of parameters." + print + print "./create_paths.py " + print + quit() + filename = sys.argv[1] + + print "Generating paths file in", filename + file_util.write_file(filename, sys.path) + print "Done" + +if __name__ == "__main__": + main() diff --git a/bundler/darwin_dyliber.py b/bundler/darwin_dyliber.py new file mode 100644 index 0000000..3a138d6 --- /dev/null +++ b/bundler/darwin_dyliber.py @@ -0,0 +1,59 @@ +import os + +from sh import otool, install_name_tool, find + +def parse_otool_output(output): + lines = output.splitlines()[1:] + libs = [] + for line in lines: + line = line.strip() + if len(line) == 0: + continue + line = line.split("(")[0].strip() + lib = os.path.split(line)[-1] + libs.append((lib, line)) + + return libs + +def locate_lib(executable_path, lib): + return find(executable_path, "-name", lib, "-type", "f").strip() + +def install_name_tooler(executable_path, lib_path): + out = otool("-L", lib_path) + _, lib_name = os.path.split(lib_path) + libs = parse_otool_output(out) + updated_any = False + for lib, original in libs: + do_id = lib == lib_name + + if original.find("Carbon") > 0: + continue + location = locate_lib(executable_path, lib) + if location is None or len(location) == 0: + continue + try: + if do_id: + install_name_tool("-id", + os.path.join("@executable_path", + os.path.relpath(location, + executable_path)), + lib_path) + else: + install_name_tool("-change", original, + os.path.join("@executable_path", + os.path.relpath(location, + executable_path)), + lib_path) + updated_any = True + except Exception as e: + print "ERROR Fixing", lib + print e + if updated_any: + print "Fixed", lib_path + +def fix_all_dylibs(executable_path): + print "Fixing all dylibs, this might take a while..." + files = find(executable_path, "-type", "f").strip().splitlines() + for f in files: + install_name_tooler(executable_path, f) + print "Done" diff --git a/bundler/depcollector.py b/bundler/depcollector.py new file mode 100644 index 0000000..de765a0 --- /dev/null +++ b/bundler/depcollector.py @@ -0,0 +1,112 @@ +import sys +import os +import errno + +from distutils import dir_util, file_util +from modulegraph import modulegraph + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: raise + +def collect_deps(root, dest_lib_dir, path_file): + mg = modulegraph.ModuleGraph([sys.path[0]] + [x.strip() for x in open(path_file, 'r').readlines()] + sys.path[1:])#, debug=3) + + mg.import_hook("distutils") + mg.import_hook("site") + mg.import_hook("jsonschema") + mg.import_hook("scrypt") + mg.import_hook("_scrypt") + mg.import_hook("ConfigParser") + mg.import_hook("Crypto") + mg.import_hook("encodings.idna") + mg.import_hook("leap.soledad.client") + mg.import_hook("leap.mail") + mg.import_hook("leap.keymanager") + mg.import_hook("argparse") + mg.import_hook("srp") + mg.import_hook("pkgutil") + mg.import_hook("pkg_resources") + mg.import_hook("_sre") + mg.import_hook("zope.proxy") + mg.run_script(root) + + packages = [mg.findNode(i) for i in ["leap.common", "leap.keymanager", "leap.mail", "leap.soledad.client", "leap.soledad.common", "jsonschema"]] + other = [] + + sorted_pkg = [(os.path.basename(mod.identifier), mod) for mod in mg.flatten()] + sorted_pkg.sort() + for (name, pkg) in sorted_pkg: + # skip namespace packages + if name == "leap" or name == "leap.soledad" or name == "google" or name == "zope" or name.endswith("leap/bitmask/app.py"): + continue + # print pkg + if isinstance(pkg, modulegraph.MissingModule): + # print "ignoring", pkg.identifier + continue + elif isinstance(pkg, modulegraph.Package): + foundpackage = False + for i in packages: + if pkg.identifier.startswith(i.identifier): + # print "skipping", pkg.identifier, "member of", i.identifier + # print " found in", i.filename + foundpackage = True + break + if foundpackage: + continue + if pkg.filename is None: + continue + if pkg not in packages: + packages.append(pkg) + else: #if isinstance(pkg, modulegraph.Extension): + foundpackage = False + for i in packages: + if pkg.identifier.startswith(i.identifier): + # print "skipping", pkg.identifier, "member of", i.identifier + # print " found in", i.filename + foundpackage = True + break + if foundpackage: + continue + if pkg.filename is None: + continue + other.append(pkg) + # print pkg.identifier + #import pdb; pdb.set_trace() + + print "Packages", len(packages) + for i in sorted(packages): + # if i.identifier == "distutils": + # i.filename = distutils.__file__ + print i.identifier, i.filename + parts = i.identifier.split(".") + destdir = os.path.join(*([dest_lib_dir]+parts)) + mkdir_p(destdir) + dir_util.copy_tree(os.path.dirname(i.filename), destdir) + before = [] + for part in parts: + before.append(part) + current = before + ["__init__.py"] + try: + with open(os.path.join(dest_lib_dir, *current), 'a'): + pass + except Exception: + pass + + print "Other", len(other) + for i in sorted(other): + # if i.identifier == "site": + # i.filename = site.__file__ + print i.identifier, i.filename + file_util.copy_file(i.filename, dest_lib_dir) + + # TODO: remove everything in dest_lib_dir/PySide that is not QtCore, QtGui and __init__ + + # ON OSX ONLY: + # TODO: remove dest_lib_dir/sre* and PYTHONHOME from the launcher script + # remove dest_lib_dir/_socket.so diff --git a/bundler/main.py b/bundler/main.py new file mode 100644 index 0000000..2c1207e --- /dev/null +++ b/bundler/main.py @@ -0,0 +1,116 @@ +# TODO: +# - Check if inside a virtualenv, and warn before doing anything +# - Build everything that we are currently expecting as a binary +# - Create complete bundle changelog + +import argparse +import os +import tempfile + +from contextlib import contextmanager +from distutils import dir_util + +from actions import GitCloneAll, PythonSetupAll, CreateDirStructure +from actions import CollectAllDeps, CopyBinaries, PLister, SeededConfig +from actions import DarwinLauncher, CopyAssets, CopyMisc, FixDylibs +from actions import DmgIt, PycRemover + +from utils import IS_MAC + +sorted_repos = [ + "leap_assets", + "leap_pycommon", + "keymanager", + "soledad", + "leap_mail", + "bitmask_client", + "bitmask_launcher", + "thandy" +] + +@contextmanager +def new_build_dir(default=None): + bd = default + if bd is None: + bd = tempfile.mkdtemp(prefix="bundler-") + yield bd + # Only remove if created a temp dir + if default is None: + dir_util.remove_tree(bd) + +def main(): + parser = argparse.ArgumentParser(description='Bundle creation tool.') + parser.add_argument('--workon', help="") + parser.add_argument('--skip', nargs="*", default=[], help="") + parser.add_argument('--do', nargs="*", default=[], help="") + parser.add_argument('--paths-file', help="") + parser.add_argument('--binaries', help="") + parser.add_argument('--seeded-config', help="") + parser.add_argument('--nightly', action="store_true", help="") + + args = parser.parse_args() + + assert args.paths_file is not None, \ + "We need a paths file, otherwise you'll get " \ + "problems with distutils and site" + paths_file = os.path.realpath(args.paths_file) + + assert args.binaries is not None, \ + "We don't support building from source, so you'll need to " \ + "specify a binaries path" + binaries_path = os.path.realpath(args.binaries) + + seeded_config = None + if args.seeded_config is not None: + seeded_config = os.path.realpath(args.seeded_config) + + with new_build_dir(os.path.realpath(args.workon)) as bd: + print "Doing it all in", bd + + def init(t, bd=bd): + return t(bd, args.skip, args.do) + + gc = init(GitCloneAll) + gc.run(sorted_repos, args.nightly) + + ps = init(PythonSetupAll) + ps.run(sorted_repos) + + cd = init(CreateDirStructure, os.path.join(bd, "Bitmask")) + cd.run() + + dp = init(CollectAllDeps) + dp.run(paths_file) + + if binaries_path is not None: + cb = init(CopyBinaries) + cb.run(binaries_path) + + if seeded_config is not None: + sc = init(SeededConfig) + sc.run(seeded_config) + + if IS_MAC: + pl = init(PLister) + pl.run() + dl = init(DarwinLauncher) + dl.run() + ca = init(CopyAssets) + ca.run() + fd = init(FixDylibs) + fd.run() + + cm = init(CopyMisc) + cm.run() + + pyc = init(PycRemover) + pyc.run() + + if IS_MAC: + dm = init(DmgIt) + dm.run() + + # do manifest on windows + +if __name__ == "__main__": + main() diff --git a/bundler/utils.py b/bundler/utils.py new file mode 100644 index 0000000..3852ee7 --- /dev/null +++ b/bundler/utils.py @@ -0,0 +1,3 @@ +import sys + +IS_MAC = sys.platform == "darwin" -- cgit v1.2.3