From c06bf409d3fe2268011214301c59dc1c279fae44 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 4 Feb 2015 00:59:32 -0400 Subject: create sumo tarball with leap dependencies (Closes: #6683) --- setup.py | 202 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 185 insertions(+), 17 deletions(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index bfd7eff0..4ce19c09 100755 --- a/setup.py +++ b/setup.py @@ -99,20 +99,7 @@ class freeze_debianver(Command): To be used after merging the development branch onto the debian one. """ user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - proceed = str(raw_input( - "This will overwrite the file _version.py. Continue? [y/N] ")) - if proceed != "y": - print("He. You scared. Aborting.") - return - template = r""" + template = r""" # This file was generated by the `freeze_debianver` command in setup.py # Using 'versioneer.py' (0.7+) from # revision-control system data, or from the parent directory name of an @@ -122,18 +109,43 @@ class freeze_debianver(Command): version_version = '{version}' version_full = '{version_full}' """ - templatefun = r""" + templatefun = r""" def get_versions(default={}, verbose=False): return {'version': version_version, 'full': version_full} """ - subst_template = template.format( + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + proceed = str(raw_input( + "This will overwrite the file _version.py. Continue? [y/N] ")) + if proceed != "y": + print("He. You scared. Aborting.") + return + subst_template = self.template.format( version=VERSION_SHORT, - version_full=VERSION_FULL) + templatefun + version_full=VERSION_FULL) + self.templatefun with open(versioneer.versionfile_source, 'w') as f: f.write(subst_template) +def freeze_pkg_ver(path, version_short, version_full): + """ + Freeze the _version in other modules, used during the gathering of + all the leap modules in the sumo tarball. + """ + subst_template = freeze_debianver.template.format( + version=version_short, + version_full=version_full) + freeze_debianver.templatefun + with open(path, 'w') as f: + f.write(subst_template) + + cmdclass["freeze_debianver"] = freeze_debianver parsed_reqs = utils.parse_requirements() @@ -240,13 +252,169 @@ class cmd_build(versioneer_build): class cmd_sdist(versioneer_sdist): + + user_options = versioneer_sdist.user_options + \ + [('sumo', 's', + "create a 'sumo' sdist which includes the contents of all " + "the leap.* packages") + ] + boolean_options = ['sumo'] + leap_sumo_packages = ['soledad.common', 'soledad.client', + 'keymanager', 'mail', 'common'] + + def initialize_options(self): + versioneer_sdist.initialize_options(self) + self.sumo = False + def run(self): return versioneer_sdist.run(self) def make_release_tree(self, base_dir, files): versioneer_sdist.make_release_tree(self, base_dir, files) + # We need to copy the requirements to the specified path + # so that the client has a copy to do the startup checks. copy_reqs(base_dir, withsrc=True) + def make_distribution(self): + # add our extra files to the list just before building the + # tarball/zipfile. We override make_distribution() instead of run() + # because setuptools.command.sdist.run() does not lend itself to + # easy/robust subclassing (the code we need to add goes right smack + # in the middle of a 12-line method). If this were the distutils + # version, we'd override get_file_list(). + + if self.sumo: + # If '--sumo' was specified, include all the leap.* in the sdist. + vdict = _get_leap_versions() + vdict['soledad.common'] = vdict['soledad'] + vdict['soledad.client'] = vdict['soledad'] + import importlib + for module in self.leap_sumo_packages: + full_module = "leap." + module + importlib.import_module(full_module) + src_path = "src/leap/" + _fix_namespace(module) + imported_module = importlib.sys.modules[full_module] + copy_recursively( + imported_module.__path__[0] + "/", + src_path) + all_module_files = list_recursively(src_path) + self.filelist.extend(all_module_files) + freeze_pkg_ver( + src_path + "/_version.py", vdict[module], "sumo") + with open('src/leap/__init__.py', 'w') as nuke_top_init: + nuke_top_init.write('') + + # In addition, we want the tarball/zipfile to have -SUMO in the + # name, and the unpacked directory to have -SUMO too. The easiest + # way to do this is to patch self.distribution and override the + # get_fullname() method. (an alternative is to modify + # self.distribution.metadata.version, but that also affects the + # contents of PKG-INFO). + fullname = self.distribution.get_fullname() + + def get_fullname(): + return fullname + "-SUMO" + self.distribution.get_fullname = get_fullname + + try: + old_mask = os.umask(int("022", 8)) + return versioneer_sdist.make_distribution(self) + finally: + os.umask(old_mask) + for module in self.leap_sumo_packages: + # check, just in case... + if module and module != "bitmask": + shutil.rmtree("src/leap/" + _fix_namespace(module)) + + +import shutil +import glob + + +def _get_leap_versions(): + versions = {} + with open("pkg/leap_versions.txt") as vf: + lines = vf.readlines() + for line in lines: + pkg, ver = line.split('\t') + versions[pkg.strip().replace('leap_', '')] = ver.strip() + return versions + + +def _fix_namespace(path): + if path in ('soledad.common', 'soledad.client'): + return path.replace('.', '/') + return path + + +_ignore_files = ('*.pyc', '_trial*', '*.swp', '.*', 'cert', 'test*') +_ignore_dirs = ('tests', '_trial*', 'test*') +_ignore_paths = _ignore_files + _ignore_dirs + +is_excluded_path = lambda path: any( + map(lambda pattern: glob.fnmatch.fnmatch(path, pattern), + _ignore_paths)) + + +def _should_exclude(path): + folder, f = os.path.split(path) + if is_excluded_path(f): + return True + upper, leaf = os.path.split(folder) + if is_excluded_path(leaf): + return True + return False + + +def list_recursively(root_dir): + file_list = [] + for root, sub_dirs, files in os.walk(root_dir): + for f in files: + is_excluded = _should_exclude(f) + if not is_excluded: + file_list.append(os.path.join(root, f)) + return file_list + + +def _mkdir_recursively(path): + sub_path = os.path.dirname(path) + if not os.path.exists(sub_path): + _mkdir_recursively(sub_path) + if not os.path.exists(path): + os.mkdir(path) + + +def copy_recursively(source_folder, destination_folder): + + if not os.path.exists(destination_folder): + _mkdir_recursively(destination_folder) + + for root, dirs, files in os.walk(source_folder): + if _should_exclude(root): + continue + for item in files: + if _should_exclude(item): + continue + src_path = os.path.join(root, item) + dst_path = os.path.join( + destination_folder, src_path.replace(source_folder, "")) + if _should_exclude(dst_path): + continue + if os.path.exists(dst_path): + if os.stat(src_path).st_mtime > os.stat(dst_path).st_mtime: + shutil.copy2(src_path, dst_path) + else: + shutil.copy2(src_path, dst_path) + for item in dirs: + if _should_exclude(item): + continue + src_path = os.path.join(root, item) + dst_path = os.path.join( + destination_folder, src_path.replace(source_folder, "")) + if _should_exclude(dst_path): + continue + if not os.path.exists(dst_path): + os.mkdir(dst_path) cmdclass["build"] = cmd_build cmdclass["sdist"] = cmd_sdist -- cgit v1.2.3 From c3919c78af24e7a29653341fe631781ae47fae3d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 21 Jul 2015 00:52:27 -0400 Subject: [pkg] add empty __init__ on soledad folder otherwise the module cannot be imported (in the pseudo-pkg produced by the sumo tarball) - Releases: 0.9.0 --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 4ce19c09..33616f5e 100755 --- a/setup.py +++ b/setup.py @@ -274,6 +274,14 @@ class cmd_sdist(versioneer_sdist): # We need to copy the requirements to the specified path # so that the client has a copy to do the startup checks. copy_reqs(base_dir, withsrc=True) + with open(os.path.join(base_dir, + 'src', 'leap', '__init__.py'), + 'w') as nuke_top_init: + nuke_top_init.write('') + with open(os.path.join(base_dir, + 'src', 'leap', 'soledad', '__init__.py'), + 'w') as nuke_soledad_ns: + nuke_soledad_ns.write('') def make_distribution(self): # add our extra files to the list just before building the @@ -301,8 +309,6 @@ class cmd_sdist(versioneer_sdist): self.filelist.extend(all_module_files) freeze_pkg_ver( src_path + "/_version.py", vdict[module], "sumo") - with open('src/leap/__init__.py', 'w') as nuke_top_init: - nuke_top_init.write('') # In addition, we want the tarball/zipfile to have -SUMO in the # name, and the unpacked directory to have -SUMO too. The easiest -- cgit v1.2.3 From 7482e04ab1e75ba4b3d98a1129f8d044919a4b02 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 22 Jul 2015 16:50:07 -0400 Subject: [pkg] separate leap requirements this is part of a process to make the setup of the development mode less troublesome. from now on, setting up a virtualenv in pure development mode will be as easy as telling pip to just install the external dependencies:: pip install -r pkg/requirements.pip and traversing all the leap repos for the needed leap dependencies doing:: python setup.py develop - Related: #7288 --- setup.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 33616f5e..6a4bced3 100755 --- a/setup.py +++ b/setup.py @@ -149,6 +149,21 @@ def freeze_pkg_ver(path, version_short, version_full): cmdclass["freeze_debianver"] = freeze_debianver parsed_reqs = utils.parse_requirements() +if utils.is_develop_mode(): + print("") + print ("[WARNING] Skipping leap-specific dependencies " + "because development mode is detected.") + print ("[WARNING] You can install " + "the latest published versions with " + "'pip install -r pkg/requirements-leap.pip'") + print ("[WARNING] Or you can instead do 'python setup.py develop' " + "from the parent folder of each one of them.") + print("") +else: + parsed_reqs += utils.parse_requirements( + reqfiles=["pkg/requirements-leap.pip"]) + + leap_launcher = 'bitmask=leap.bitmask.app:start_app' from setuptools.command.develop import develop as _develop @@ -468,6 +483,7 @@ if IS_MAC: import py2app.recipes py2app.recipes.jsonschema = jsonschema_recipe() + setup( name="leap.bitmask", package_dir={"": "src"}, -- cgit v1.2.3 From 4f4c50f159ea191e12abf3165157fb112c4e985e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 22 Jul 2015 16:55:29 -0400 Subject: [bug] do not attempt to install data files inside a venv --- setup.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 6a4bced3..1a6a7edf 100755 --- a/setup.py +++ b/setup.py @@ -447,15 +447,20 @@ IS_MAC = _system == "Darwin" data_files = [] + if IS_LINUX: # XXX use check_for_permissions to install data # globally. Or make specific install command. See #3805 - data_files = [ - ("share/polkit-1/actions", - ["pkg/linux/polkit/se.leap.bitmask.policy"]), - ("/usr/sbin", - ["pkg/linux/bitmask-root"]), - ] + isset = lambda var: os.environ.get(var, None) + if isset('VIRTUAL_ENV') or isset('LEAP_SKIP_INIT'): + data_files = None + else: + data_files = [ + ("share/polkit-1/actions", + ["pkg/linux/polkit/se.leap.bitmask.policy"]), + ("/usr/sbin", + ["pkg/linux/bitmask-root"]), + ] extra_options = {} -- cgit v1.2.3 From 7baaa7689f094c5e82f8b6d2d9bd400a7a102728 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 5 Aug 2015 19:07:42 -0700 Subject: [pkg] remove py2app snippet from setup.py thanks for making our lives easier during all this time, now we're starting a relationship with pyinstaller. maybe our roads cross again in the future. --- setup.py | 25 ------------------------- 1 file changed, 25 deletions(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 1a6a7edf..1e77ccf9 100755 --- a/setup.py +++ b/setup.py @@ -464,31 +464,6 @@ if IS_LINUX: extra_options = {} -if IS_MAC: - extra_options["app"] = ['src/leap/bitmask/app.py'] - OPTIONS = { - 'argv_emulation': True, - 'plist': 'pkg/osx/Info.plist', - 'iconfile': 'pkg/osx/bitmask.icns', - } - extra_options["options"] = {'py2app': OPTIONS} - extra_options["setup_requires"] = ['py2app'] - - class jsonschema_recipe(object): - def check(self, dist, mf): - m = mf.findNode('jsonschema') - if m is None: - return None - - # Don't put jsonschema in the site-packages.zip file - return dict( - packages=['jsonschema'] - ) - - import py2app.recipes - py2app.recipes.jsonschema = jsonschema_recipe() - - setup( name="leap.bitmask", package_dir={"": "src"}, -- cgit v1.2.3 From 4f0f6ec24e0e6e1d27509130c0aae3b818f19195 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 12 Aug 2015 12:19:00 -0400 Subject: [bug] freeze also bitmask module version in sumo tarball if __version__ is not properly assigned, provider tests were breaking when parsing BITMASK_VERSION (which is None, where a string is expected). - Resolves: #7322 --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 1e77ccf9..16438f8a 100755 --- a/setup.py +++ b/setup.py @@ -322,8 +322,13 @@ class cmd_sdist(versioneer_sdist): src_path) all_module_files = list_recursively(src_path) self.filelist.extend(all_module_files) + module_ver = vdict[module] freeze_pkg_ver( - src_path + "/_version.py", vdict[module], "sumo") + src_path + "/_version.py", + module_ver, "%s-sumo" % module_ver) + freeze_pkg_ver( + "src/leap/bitmask/_version.py", + VERSION, "%s-sumo" % VERSION) # In addition, we want the tarball/zipfile to have -SUMO in the # name, and the unpacked directory to have -SUMO too. The easiest -- cgit v1.2.3 From 848cc7bd6ce762e74e189a046480b36347e09db4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 12 Aug 2015 12:20:49 -0400 Subject: [style] autopep8 --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 16438f8a..b10189a2 100755 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ from setuptools import Command class freeze_debianver(Command): + """ Freezes the version in a debian branch. To be used after merging the development branch onto the debian one. @@ -184,6 +185,7 @@ def copy_reqs(path, withsrc=False): class cmd_develop(_develop): + def run(self): # versioneer: versions = versioneer.get_versions(verbose=True) @@ -198,6 +200,7 @@ cmdclass["develop"] = cmd_develop class cmd_binary_hash(Command): + """ Update the _binaries.py file with hashes for the different helpers. This is used from within the bundle. @@ -261,6 +264,7 @@ versioneer_sdist = cmdclass['sdist'] class cmd_build(versioneer_build): + def run(self): versioneer_build.run(self) copy_reqs(self.build_lib) -- cgit v1.2.3