create sumo tarball with leap dependencies (Closes: #6683)

6 files changed, 209 insertions, 17 deletions

diff --git a/Makefile b/Makefile
index 3ef61df9..efd27b12 100644
--- a/Makefile
+++ b/Makefile
@@ -122,5 +122,14 @@ install_wheel:
 gather_deps:
 	pipdeptree | pkg/scripts/filter-bitmask-deps
 
+all_leap_develop:
+	cd ../keymanager && git checkout develop
+	cd ../leap_common && git checkout develop
+	cd ../leap_mail && git checkout develop
+	cd ../soledad && git checkout develop
+
+all_leap_release_tags:
+	pkg/scripts/checkout_leap_versions.sh
+
 clean :
 	$(RM) $(COMPILED_UI) $(COMPILED_RESOURCES) $(COMPILED_UI:.py=.pyc) $(COMPILED_RESOURCES:.py=.pyc)
diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT
index 1eadcbe0..d5786a79 100644
--- a/changes/VERSION_COMPAT
+++ b/changes/VERSION_COMPAT
@@ -4,6 +4,9 @@
 # Add your changes here so we can properly update
 # requirements.pip during the release process.
 # (leave header when resetting)
+#
+# When bumping it during the release cycle
+# remember to update also pkg/leap_versions.txt
 #################################################
 #
 # BEGIN DEPENDENCY LIST -------------------------
diff --git a/changes/feature_sumo-tarball b/changes/feature_sumo-tarball
new file mode 100644
index 00000000..c4da0aa5
--- /dev/null
+++ b/changes/feature_sumo-tarball
@@ -0,0 +1 @@
+- Bug #6683: Add ability to generate sumo tarball.
diff --git a/pkg/leap_versions.txt b/pkg/leap_versions.txt
new file mode 100644
index 00000000..b95d1b29
--- /dev/null
+++ b/pkg/leap_versions.txt
@@ -0,0 +1,4 @@
+soledad 	0.6.3
+keymanager	0.3.8
+leap_common	0.3.9
+leap_mail	0.3.11
diff --git a/pkg/scripts/checkout_leap_versions.sh b/pkg/scripts/checkout_leap_versions.sh
new file mode 100755
index 00000000..9d6c6fab
--- /dev/null
+++ b/pkg/scripts/checkout_leap_versions.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+cat pkg/leap_versions.txt | while read line
+do
+	package=$(echo $line | cut -f1 -d' ')
+	tag=$(echo $line | cut -f2 -d' ')
+	cd ../$package && git checkout $tag
+done
diff --git a/setup.py b/setup.py
index bfd7eff0..4ce19c09 100755
--- a/setup.py
+++ b/setup.py
@@ -99,20 +99,7 @@ class freeze_debianver(Command):
     To be used after merging the development branch onto the debian one.
     """
     user_options = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        proceed = str(raw_input(
-            "This will overwrite the file _version.py. Continue? [y/N] "))
-        if proceed != "y":
-            print("He. You scared. Aborting.")
-            return
-        template = r"""
+    template = r"""
 # This file was generated by the `freeze_debianver` command in setup.py
 # Using 'versioneer.py' (0.7+) from
 # revision-control system data, or from the parent directory name of an
@@ -122,18 +109,43 @@ class freeze_debianver(Command):
 version_version = '{version}'
 version_full = '{version_full}'
 """
-        templatefun = r"""
+    templatefun = r"""
 
 def get_versions(default={}, verbose=False):
         return {'version': version_version, 'full': version_full}
 """
-        subst_template = template.format(
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        proceed = str(raw_input(
+            "This will overwrite the file _version.py. Continue? [y/N] "))
+        if proceed != "y":
+            print("He. You scared. Aborting.")
+            return
+        subst_template = self.template.format(
             version=VERSION_SHORT,
-            version_full=VERSION_FULL) + templatefun
+            version_full=VERSION_FULL) + self.templatefun
         with open(versioneer.versionfile_source, 'w') as f:
             f.write(subst_template)
 
 
+def freeze_pkg_ver(path, version_short, version_full):
+    """
+    Freeze the _version in other modules, used during the gathering of
+    all the leap modules in the sumo tarball.
+    """
+    subst_template = freeze_debianver.template.format(
+        version=version_short,
+        version_full=version_full) + freeze_debianver.templatefun
+    with open(path, 'w') as f:
+        f.write(subst_template)
+
+
 cmdclass["freeze_debianver"] = freeze_debianver
 parsed_reqs = utils.parse_requirements()
 
@@ -240,13 +252,169 @@ class cmd_build(versioneer_build):
 
 
 class cmd_sdist(versioneer_sdist):
+
+    user_options = versioneer_sdist.user_options + \
+        [('sumo', 's',
+          "create a 'sumo' sdist which includes the contents of all "
+          "the leap.* packages")
+         ]
+    boolean_options = ['sumo']
+    leap_sumo_packages = ['soledad.common', 'soledad.client',
+                          'keymanager', 'mail', 'common']
+
+    def initialize_options(self):
+        versioneer_sdist.initialize_options(self)
+        self.sumo = False
+
     def run(self):
         return versioneer_sdist.run(self)
 
     def make_release_tree(self, base_dir, files):
         versioneer_sdist.make_release_tree(self, base_dir, files)
+        # We need to copy the requirements to the specified path
+        # so that the client has a copy to do the startup checks.
         copy_reqs(base_dir, withsrc=True)
 
+    def make_distribution(self):
+        # add our extra files to the list just before building the
+        # tarball/zipfile. We override make_distribution() instead of run()
+        # because setuptools.command.sdist.run() does not lend itself to
+        # easy/robust subclassing (the code we need to add goes right smack
+        # in the middle of a 12-line method). If this were the distutils
+        # version, we'd override get_file_list().
+
+        if self.sumo:
+            # If '--sumo' was specified, include all the leap.* in the sdist.
+            vdict = _get_leap_versions()
+            vdict['soledad.common'] = vdict['soledad']
+            vdict['soledad.client'] = vdict['soledad']
+            import importlib
+            for module in self.leap_sumo_packages:
+                full_module = "leap." + module
+                importlib.import_module(full_module)
+                src_path = "src/leap/" + _fix_namespace(module)
+                imported_module = importlib.sys.modules[full_module]
+                copy_recursively(
+                    imported_module.__path__[0] + "/",
+                    src_path)
+                all_module_files = list_recursively(src_path)
+                self.filelist.extend(all_module_files)
+                freeze_pkg_ver(
+                    src_path + "/_version.py", vdict[module], "sumo")
+            with open('src/leap/__init__.py', 'w') as nuke_top_init:
+                nuke_top_init.write('')
+
+            # In addition, we want the tarball/zipfile to have -SUMO in the
+            # name, and the unpacked directory to have -SUMO too. The easiest
+            # way to do this is to patch self.distribution and override the
+            # get_fullname() method. (an alternative is to modify
+            # self.distribution.metadata.version, but that also affects the
+            # contents of PKG-INFO).
+            fullname = self.distribution.get_fullname()
+
+            def get_fullname():
+                return fullname + "-SUMO"
+            self.distribution.get_fullname = get_fullname
+
+        try:
+            old_mask = os.umask(int("022", 8))
+            return versioneer_sdist.make_distribution(self)
+        finally:
+            os.umask(old_mask)
+            for module in self.leap_sumo_packages:
+                # check, just in case...
+                if module and module != "bitmask":
+                    shutil.rmtree("src/leap/" + _fix_namespace(module))
+
+
+import shutil
+import glob
+
+
+def _get_leap_versions():
+    versions = {}
+    with open("pkg/leap_versions.txt") as vf:
+        lines = vf.readlines()
+    for line in lines:
+        pkg, ver = line.split('\t')
+        versions[pkg.strip().replace('leap_', '')] = ver.strip()
+    return versions
+
+
+def _fix_namespace(path):
+    if path in ('soledad.common', 'soledad.client'):
+        return path.replace('.', '/')
+    return path
+
+
+_ignore_files = ('*.pyc', '_trial*', '*.swp', '.*', 'cert', 'test*')
+_ignore_dirs = ('tests', '_trial*', 'test*')
+_ignore_paths = _ignore_files + _ignore_dirs
+
+is_excluded_path = lambda path: any(
+    map(lambda pattern: glob.fnmatch.fnmatch(path, pattern),
+        _ignore_paths))
+
+
+def _should_exclude(path):
+    folder, f = os.path.split(path)
+    if is_excluded_path(f):
+        return True
+    upper, leaf = os.path.split(folder)
+    if is_excluded_path(leaf):
+        return True
+    return False
+
+
+def list_recursively(root_dir):
+    file_list = []
+    for root, sub_dirs, files in os.walk(root_dir):
+        for f in files:
+            is_excluded = _should_exclude(f)
+            if not is_excluded:
+                file_list.append(os.path.join(root, f))
+    return file_list
+
+
+def _mkdir_recursively(path):
+    sub_path = os.path.dirname(path)
+    if not os.path.exists(sub_path):
+        _mkdir_recursively(sub_path)
+    if not os.path.exists(path):
+        os.mkdir(path)
+
+
+def copy_recursively(source_folder, destination_folder):
+
+    if not os.path.exists(destination_folder):
+        _mkdir_recursively(destination_folder)
+
+    for root, dirs, files in os.walk(source_folder):
+        if _should_exclude(root):
+            continue
+        for item in files:
+            if _should_exclude(item):
+                continue
+            src_path = os.path.join(root, item)
+            dst_path = os.path.join(
+                destination_folder, src_path.replace(source_folder, ""))
+            if _should_exclude(dst_path):
+                continue
+            if os.path.exists(dst_path):
+                if os.stat(src_path).st_mtime > os.stat(dst_path).st_mtime:
+                    shutil.copy2(src_path, dst_path)
+            else:
+                shutil.copy2(src_path, dst_path)
+        for item in dirs:
+            if _should_exclude(item):
+                continue
+            src_path = os.path.join(root, item)
+            dst_path = os.path.join(
+                destination_folder, src_path.replace(source_folder, ""))
+            if _should_exclude(dst_path):
+                continue
+            if not os.path.exists(dst_path):
+                os.mkdir(dst_path)
 
 cmdclass["build"] = cmd_build
 cmdclass["sdist"] = cmd_sdist