From 9070d74a47158f5749c5e16b8d9d9e62a55d07df Mon Sep 17 00:00:00 2001 From: Micah Anderson Date: Thu, 15 Aug 2013 10:12:25 -0400 Subject: regenerate things with protobuf 2.4.1 --- src/leap/common/events/events_pb2.py | 136 +++++++++++++++++------------------ 1 file changed, 65 insertions(+), 71 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/events_pb2.py b/src/leap/common/events/events_pb2.py index 274514c..e25c7da 100644 --- a/src/leap/common/events/events_pb2.py +++ b/src/leap/common/events/events_pb2.py @@ -1,76 +1,74 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: events.proto -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import service as _service +from google.protobuf import descriptor +from google.protobuf import message +from google.protobuf import reflection +from google.protobuf import service from google.protobuf import service_reflection from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) -DESCRIPTOR = _descriptor.FileDescriptor( +DESCRIPTOR = descriptor.FileDescriptor( name='events.proto', package='leap.common.events', serialized_pb='\n\x0c\x65vents.proto\x12\x12leap.common.events\"\x97\x01\n\rSignalRequest\x12(\n\x05\x65vent\x18\x01 \x02(\x0e\x32\x19.leap.common.events.Event\x12\x0f\n\x07\x63ontent\x18\x02 \x02(\t\x12\x12\n\nmac_method\x18\x03 \x02(\t\x12\x0b\n\x03mac\x18\x04 \x02(\x0c\x12\x12\n\nenc_method\x18\x05 \x01(\t\x12\x16\n\x0e\x65rror_occurred\x18\x06 \x01(\x08\"j\n\x0fRegisterRequest\x12(\n\x05\x65vent\x18\x01 \x02(\x0e\x32\x19.leap.common.events.Event\x12\x0c\n\x04port\x18\x02 \x02(\x05\x12\x12\n\nmac_method\x18\x03 \x02(\t\x12\x0b\n\x03mac\x18\x04 \x02(\x0c\"l\n\x11UnregisterRequest\x12(\n\x05\x65vent\x18\x01 \x02(\x0e\x32\x19.leap.common.events.Event\x12\x0c\n\x04port\x18\x02 \x02(\x05\x12\x12\n\nmac_method\x18\x03 \x02(\t\x12\x0b\n\x03mac\x18\x04 \x02(\x0c\"\r\n\x0bPingRequest\"\x82\x01\n\rEventResponse\x12\x38\n\x06status\x18\x01 \x02(\x0e\x32(.leap.common.events.EventResponse.Status\x12\x0e\n\x06result\x18\x02 \x01(\t\"\'\n\x06Status\x12\x06\n\x02OK\x10\x01\x12\n\n\x06UNAUTH\x10\x02\x12\t\n\x05\x45RROR\x10\x03*\xe7\x02\n\x05\x45vent\x12\x15\n\x11\x43LIENT_SESSION_ID\x10\x01\x12\x0e\n\nCLIENT_UID\x10\x02\x12\x19\n\x15SOLEDAD_CREATING_KEYS\x10\x03\x12\x1e\n\x1aSOLEDAD_DONE_CREATING_KEYS\x10\x04\x12\x1a\n\x16SOLEDAD_UPLOADING_KEYS\x10\x05\x12\x1f\n\x1bSOLEDAD_DONE_UPLOADING_KEYS\x10\x06\x12\x1c\n\x18SOLEDAD_DOWNLOADING_KEYS\x10\x07\x12!\n\x1dSOLEDAD_DONE_DOWNLOADING_KEYS\x10\x08\x12\x1c\n\x18SOLEDAD_NEW_DATA_TO_SYNC\x10\t\x12\x1a\n\x16SOLEDAD_DONE_DATA_SYNC\x10\n\x12\x17\n\x13UPDATER_NEW_UPDATES\x10\x0b\x12\x19\n\x15UPDATER_DONE_UPDATING\x10\x0c\x12\x10\n\x0cRAISE_WINDOW\x10\r2\xdd\x02\n\x13\x45ventsServerService\x12J\n\x04ping\x12\x1f.leap.common.events.PingRequest\x1a!.leap.common.events.EventResponse\x12R\n\x08register\x12#.leap.common.events.RegisterRequest\x1a!.leap.common.events.EventResponse\x12V\n\nunregister\x12%.leap.common.events.UnregisterRequest\x1a!.leap.common.events.EventResponse\x12N\n\x06signal\x12!.leap.common.events.SignalRequest\x1a!.leap.common.events.EventResponse2\xb1\x01\n\x13\x45ventsClientService\x12J\n\x04ping\x12\x1f.leap.common.events.PingRequest\x1a!.leap.common.events.EventResponse\x12N\n\x06signal\x12!.leap.common.events.SignalRequest\x1a!.leap.common.events.EventResponseB\x03\x90\x01\x01') -_EVENT = _descriptor.EnumDescriptor( +_EVENT = descriptor.EnumDescriptor( name='Event', full_name='leap.common.events.Event', filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='CLIENT_SESSION_ID', index=0, number=1, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='CLIENT_UID', index=1, number=2, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='SOLEDAD_CREATING_KEYS', index=2, number=3, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='SOLEDAD_DONE_CREATING_KEYS', index=3, number=4, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='SOLEDAD_UPLOADING_KEYS', index=4, number=5, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='SOLEDAD_DONE_UPLOADING_KEYS', index=5, number=6, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='SOLEDAD_DOWNLOADING_KEYS', index=6, number=7, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='SOLEDAD_DONE_DOWNLOADING_KEYS', index=7, number=8, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='SOLEDAD_NEW_DATA_TO_SYNC', index=8, number=9, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='SOLEDAD_DONE_DATA_SYNC', index=9, number=10, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='UPDATER_NEW_UPDATES', index=10, number=11, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='UPDATER_DONE_UPDATING', index=11, number=12, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='RAISE_WINDOW', index=12, number=13, options=None, type=None), @@ -81,7 +79,7 @@ _EVENT = _descriptor.EnumDescriptor( serialized_end=916, ) -Event = enum_type_wrapper.EnumTypeWrapper(_EVENT) + CLIENT_SESSION_ID = 1 CLIENT_UID = 2 SOLEDAD_CREATING_KEYS = 3 @@ -97,21 +95,21 @@ UPDATER_DONE_UPDATING = 12 RAISE_WINDOW = 13 -_EVENTRESPONSE_STATUS = _descriptor.EnumDescriptor( +_EVENTRESPONSE_STATUS = descriptor.EnumDescriptor( name='Status', full_name='leap.common.events.EventResponse.Status', filename=None, file=DESCRIPTOR, values=[ - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='OK', index=0, number=1, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='UNAUTH', index=1, number=2, options=None, type=None), - _descriptor.EnumValueDescriptor( + descriptor.EnumValueDescriptor( name='ERROR', index=2, number=3, options=None, type=None), @@ -123,49 +121,49 @@ _EVENTRESPONSE_STATUS = _descriptor.EnumDescriptor( ) -_SIGNALREQUEST = _descriptor.Descriptor( +_SIGNALREQUEST = descriptor.Descriptor( name='SignalRequest', full_name='leap.common.events.SignalRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='event', full_name='leap.common.events.SignalRequest.event', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=1, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='content', full_name='leap.common.events.SignalRequest.content', index=1, number=2, type=9, cpp_type=9, label=2, has_default_value=False, default_value=unicode("", "utf-8"), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='mac_method', full_name='leap.common.events.SignalRequest.mac_method', index=2, number=3, type=9, cpp_type=9, label=2, has_default_value=False, default_value=unicode("", "utf-8"), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='mac', full_name='leap.common.events.SignalRequest.mac', index=3, number=4, type=12, cpp_type=9, label=2, has_default_value=False, default_value="", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='enc_method', full_name='leap.common.events.SignalRequest.enc_method', index=4, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=unicode("", "utf-8"), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='error_occurred', full_name='leap.common.events.SignalRequest.error_occurred', index=5, number=6, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, @@ -186,35 +184,35 @@ _SIGNALREQUEST = _descriptor.Descriptor( ) -_REGISTERREQUEST = _descriptor.Descriptor( +_REGISTERREQUEST = descriptor.Descriptor( name='RegisterRequest', full_name='leap.common.events.RegisterRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='event', full_name='leap.common.events.RegisterRequest.event', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=1, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='port', full_name='leap.common.events.RegisterRequest.port', index=1, number=2, type=5, cpp_type=1, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='mac_method', full_name='leap.common.events.RegisterRequest.mac_method', index=2, number=3, type=9, cpp_type=9, label=2, has_default_value=False, default_value=unicode("", "utf-8"), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='mac', full_name='leap.common.events.RegisterRequest.mac', index=3, number=4, type=12, cpp_type=9, label=2, has_default_value=False, default_value="", @@ -235,35 +233,35 @@ _REGISTERREQUEST = _descriptor.Descriptor( ) -_UNREGISTERREQUEST = _descriptor.Descriptor( +_UNREGISTERREQUEST = descriptor.Descriptor( name='UnregisterRequest', full_name='leap.common.events.UnregisterRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='event', full_name='leap.common.events.UnregisterRequest.event', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=1, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='port', full_name='leap.common.events.UnregisterRequest.port', index=1, number=2, type=5, cpp_type=1, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='mac_method', full_name='leap.common.events.UnregisterRequest.mac_method', index=2, number=3, type=9, cpp_type=9, label=2, has_default_value=False, default_value=unicode("", "utf-8"), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='mac', full_name='leap.common.events.UnregisterRequest.mac', index=3, number=4, type=12, cpp_type=9, label=2, has_default_value=False, default_value="", @@ -284,7 +282,7 @@ _UNREGISTERREQUEST = _descriptor.Descriptor( ) -_PINGREQUEST = _descriptor.Descriptor( +_PINGREQUEST = descriptor.Descriptor( name='PingRequest', full_name='leap.common.events.PingRequest', filename=None, @@ -305,21 +303,21 @@ _PINGREQUEST = _descriptor.Descriptor( ) -_EVENTRESPONSE = _descriptor.Descriptor( +_EVENTRESPONSE = descriptor.Descriptor( name='EventResponse', full_name='leap.common.events.EventResponse', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='status', full_name='leap.common.events.EventResponse.status', index=0, number=1, type=14, cpp_type=8, label=2, has_default_value=False, default_value=1, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( + descriptor.FieldDescriptor( name='result', full_name='leap.common.events.EventResponse.result', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=unicode("", "utf-8"), @@ -352,46 +350,42 @@ DESCRIPTOR.message_types_by_name['PingRequest'] = _PINGREQUEST DESCRIPTOR.message_types_by_name['EventResponse'] = _EVENTRESPONSE -class SignalRequest(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType +class SignalRequest(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType DESCRIPTOR = _SIGNALREQUEST # @@protoc_insertion_point(class_scope:leap.common.events.SignalRequest) -class RegisterRequest(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType +class RegisterRequest(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType DESCRIPTOR = _REGISTERREQUEST # @@protoc_insertion_point(class_scope:leap.common.events.RegisterRequest) -class UnregisterRequest(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType +class UnregisterRequest(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType DESCRIPTOR = _UNREGISTERREQUEST # @@protoc_insertion_point(class_scope:leap.common.events.UnregisterRequest) -class PingRequest(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType +class PingRequest(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType DESCRIPTOR = _PINGREQUEST # @@protoc_insertion_point(class_scope:leap.common.events.PingRequest) -class EventResponse(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType +class EventResponse(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType DESCRIPTOR = _EVENTRESPONSE # @@protoc_insertion_point(class_scope:leap.common.events.EventResponse) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions( - descriptor_pb2.FileOptions(), '\220\001\001') - -_EVENTSSERVERSERVICE = _descriptor.ServiceDescriptor( +_EVENTSSERVERSERVICE = descriptor.ServiceDescriptor( name='EventsServerService', full_name='leap.common.events.EventsServerService', file=DESCRIPTOR, @@ -400,7 +394,7 @@ _EVENTSSERVERSERVICE = _descriptor.ServiceDescriptor( serialized_start=919, serialized_end=1268, methods=[ - _descriptor.MethodDescriptor( + descriptor.MethodDescriptor( name='ping', full_name='leap.common.events.EventsServerService.ping', index=0, @@ -409,7 +403,7 @@ _EVENTSSERVERSERVICE = _descriptor.ServiceDescriptor( output_type=_EVENTRESPONSE, options=None, ), - _descriptor.MethodDescriptor( + descriptor.MethodDescriptor( name='register', full_name='leap.common.events.EventsServerService.register', index=1, @@ -418,7 +412,7 @@ _EVENTSSERVERSERVICE = _descriptor.ServiceDescriptor( output_type=_EVENTRESPONSE, options=None, ), - _descriptor.MethodDescriptor( + descriptor.MethodDescriptor( name='unregister', full_name='leap.common.events.EventsServerService.unregister', index=2, @@ -427,7 +421,7 @@ _EVENTSSERVERSERVICE = _descriptor.ServiceDescriptor( output_type=_EVENTRESPONSE, options=None, ), - _descriptor.MethodDescriptor( + descriptor.MethodDescriptor( name='signal', full_name='leap.common.events.EventsServerService.signal', index=3, @@ -439,7 +433,7 @@ _EVENTSSERVERSERVICE = _descriptor.ServiceDescriptor( ]) -class EventsServerService(_service.Service): +class EventsServerService(service.Service): __metaclass__ = service_reflection.GeneratedServiceType DESCRIPTOR = _EVENTSSERVERSERVICE @@ -449,7 +443,7 @@ class EventsServerService_Stub(EventsServerService): DESCRIPTOR = _EVENTSSERVERSERVICE -_EVENTSCLIENTSERVICE = _descriptor.ServiceDescriptor( +_EVENTSCLIENTSERVICE = descriptor.ServiceDescriptor( name='EventsClientService', full_name='leap.common.events.EventsClientService', file=DESCRIPTOR, @@ -458,7 +452,7 @@ _EVENTSCLIENTSERVICE = _descriptor.ServiceDescriptor( serialized_start=1271, serialized_end=1448, methods=[ - _descriptor.MethodDescriptor( + descriptor.MethodDescriptor( name='ping', full_name='leap.common.events.EventsClientService.ping', index=0, @@ -467,7 +461,7 @@ _EVENTSCLIENTSERVICE = _descriptor.ServiceDescriptor( output_type=_EVENTRESPONSE, options=None, ), - _descriptor.MethodDescriptor( + descriptor.MethodDescriptor( name='signal', full_name='leap.common.events.EventsClientService.signal', index=1, @@ -479,7 +473,7 @@ _EVENTSCLIENTSERVICE = _descriptor.ServiceDescriptor( ]) -class EventsClientService(_service.Service): +class EventsClientService(service.Service): __metaclass__ = service_reflection.GeneratedServiceType DESCRIPTOR = _EVENTSCLIENTSERVICE -- cgit v1.2.3 From 74787c2cf709356804bc880ac34a4dd0868ef05d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 30 Oct 2013 15:47:21 -0200 Subject: 0.3.4 version freeze --- src/leap/common/_version.py | 208 ++------------------------------------------ 1 file changed, 9 insertions(+), 199 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index 597e2e4..d372bfe 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -1,203 +1,13 @@ -IN_LONG_VERSION_PY = True -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.7+) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. -# This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) +version_version = '0.3.4' +version_full = '34dcb3378c444f7ef195aa8073f01710fd4b7e2b' -# these strings will be replaced by git during git-archive -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" - - -import subprocess -import sys - -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] - if verbose: - print("unable to run %s" % args[0]) - print(e) - return None - stdout = p.communicate()[0].strip() - if sys.version >= '3': - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % args[0]) - return None - return stdout - - -import sys -import re -import os.path - -def get_expanded_variables(versionfile_source): - # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} - try: - f = open(versionfile_source,"r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - variables["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - variables["full"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return variables - -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id - if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. - - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname(here) - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %s" % root) - return {} - - GIT = "git" - if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope - # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) - - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "" -parentdir_prefix = "leap.common-" -versionfile_source = "src/leap/common/_version.py" - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver +def get_versions(default={}, verbose=False): + return {'version': version_version, 'full': version_full} -- cgit v1.2.3 From 10c0e1d3456c3f8ef3950d7e11b4a4548e771073 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 5 Nov 2013 11:09:07 -0200 Subject: freeze debian ver to 0.3.5 --- src/leap/common/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index d372bfe..940759b 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,8 +5,8 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.3.4' -version_full = '34dcb3378c444f7ef195aa8073f01710fd4b7e2b' +version_version = '0.3.5' +version_full = 'fac6becd50e2f8bb9efc354fa63c65f2c1445ecc' def get_versions(default={}, verbose=False): -- cgit v1.2.3 From 4e77d18550758e54982b40e32b8f762558504312 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 10 Dec 2013 15:43:54 -0400 Subject: freeze debian version --- src/leap/common/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index 940759b..acaa91d 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,8 +5,8 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.3.5' -version_full = 'fac6becd50e2f8bb9efc354fa63c65f2c1445ecc' +version_version = '0.3.6' +version_full = '3ea9dd9c8658f9faafd1f1a9a8c8ce16e6579f4a' def get_versions(default={}, verbose=False): -- cgit v1.2.3 From bfdad9b9888f62a5849cab3830ebfc040228d270 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Dec 2013 02:32:51 -0400 Subject: freeze debian version to 0.3.7-rc --- src/leap/common/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index acaa91d..a68d311 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,8 +5,8 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.3.6' -version_full = '3ea9dd9c8658f9faafd1f1a9a8c8ce16e6579f4a' +version_version = '0.3.6-rc1' +version_full = '7afebec82d45765dde145e0e96dc50b4a34c78ec' def get_versions(default={}, verbose=False): -- cgit v1.2.3 From f15cf332f1fa78f467253666485a0953590289a7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 23 Dec 2013 11:32:33 -0400 Subject: Fix the reported string under _version (0.3.7-rc) --- src/leap/common/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index a68d311..6511e15 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,7 +5,7 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.3.6-rc1' +version_version = '0.3.7-rc1' version_full = '7afebec82d45765dde145e0e96dc50b4a34c78ec' -- cgit v1.2.3 From 14500565c0bc2461f45f216373cb66cdb750ac2b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 9 Apr 2014 14:28:55 -0500 Subject: freeze debian ver to 0.3.7 --- src/leap/common/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index 6511e15..81a674f 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,7 +5,7 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.3.7-rc1' +version_version = '0.3.7' version_full = '7afebec82d45765dde145e0e96dc50b4a34c78ec' -- cgit v1.2.3 From a33bff3701c553c7ddaa1521ae1a077e1d3dadca Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 10 Jun 2014 09:24:52 -0500 Subject: freeze debian version --- src/leap/common/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index 81a674f..8b8926f 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,8 +5,8 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.3.7' -version_full = '7afebec82d45765dde145e0e96dc50b4a34c78ec' +version_version = '0.3.8' +version_full = '0a87e2bd3c79409ef6b30cef085d90a8ca4f9441' def get_versions(default={}, verbose=False): -- cgit v1.2.3 From 42e1da0732f16592ae13de1b83061e143aec0ad9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 7 Nov 2014 18:49:45 +0100 Subject: freeze debian version --- src/leap/common/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index 8b8926f..e926143 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,8 +5,8 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.3.8' -version_full = '0a87e2bd3c79409ef6b30cef085d90a8ca4f9441' +version_version = '0.3.9' +version_full = '3de1eeba83d50793283d65ba4566dd1611ee9d4b' def get_versions(default={}, verbose=False): -- cgit v1.2.3 From c071c69e1b5a0d897674a1f7adc6ff32f19400ff Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 27 May 2015 12:49:44 -0300 Subject: [bug] Use BrowserLikePolicyForHTTPS for checking While testing the way that its implemented now, I found out that no check is being made on certificate attributes against the host. I found this simple way of creating a BrowserLikePolicyForHTTPS using a self signed cert and it worked on my test. I used test_https from Soledad for checking this (which we are fixing on another branch). Also, we don't want to depend on twisted for other things than leap.common.http. --- src/leap/common/certs.py | 17 ++++++++++++ src/leap/common/http.py | 70 +++++++++++++----------------------------------- 2 files changed, 35 insertions(+), 52 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/certs.py b/src/leap/common/certs.py index db513f6..c8e0743 100644 --- a/src/leap/common/certs.py +++ b/src/leap/common/certs.py @@ -178,3 +178,20 @@ def should_redownload(certfile, now=time.gmtime): return True return False + + +def get_compatible_ssl_context_factory(cert_path=None): + import twisted + cert = None + if twisted.version.base() > '14.0.1': + from twisted.web.client import BrowserLikePolicyForHTTPS + from twisted.internet import ssl + if cert_path: + cert = ssl.Certificate.loadPEM(open(cert_path).read()) + policy = BrowserLikePolicyForHTTPS(cert) + return policy + else: + raise Exception((""" + Twisted 14.0.2 is needed in order to have secure Client Web SSL Contexts, not %s + See: http://twistedmatrix.com/trac/ticket/7647 + """) % (twisted.version.base())) diff --git a/src/leap/common/http.py b/src/leap/common/http.py index 39f01ba..1dc5642 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -18,22 +18,28 @@ Twisted HTTP/HTTPS client. """ -import os +try: + import twisted +except ImportError: + print "*******" + print "Twisted is needed to use leap.common.http module" + print "" + print "Install the extra requirement of the package:" + print "$ pip install leap.common[Twisted]" + import sys + sys.exit(1) -from zope.interface import implements -from OpenSSL.crypto import load_certificate -from OpenSSL.crypto import FILETYPE_PEM +from leap.common.certs import get_compatible_ssl_context_factory + +from zope.interface import implements from twisted.internet import reactor -from twisted.internet.ssl import ClientContextFactory -from twisted.internet.ssl import CertificateOptions from twisted.internet.defer import succeed from twisted.web.client import Agent from twisted.web.client import HTTPConnectionPool from twisted.web.client import readBody -from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer @@ -55,33 +61,12 @@ class HTTPClient(object): self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = 10 - if cert_file: - cert = self._load_cert(cert_file) - self._agent = Agent( - reactor, - HTTPClient.ClientContextFactory(cert), - pool=self._pool) - else: - # trust the system's CAs - self._agent = Agent( - reactor, - BrowserLikePolicyForHTTPS(), - pool=self._pool) - - def _load_cert(self, cert_file): - """ - Load a X509 certificate from a file. - - :param cert_file: The path to the certificate file. - :type cert_file: str + policy = get_compatible_ssl_context_factory(cert_file) - :return: The X509 certificate. - :rtype: OpenSSL.crypto.X509 - """ - if os.path.exists(cert_file): - with open(cert_file) as f: - data = f.read() - return load_certificate(FILETYPE_PEM, data) + self._agent = Agent( + reactor, + policy, + pool=self._pool) def request(self, url, method='GET', body=None, headers={}): """ @@ -106,25 +91,6 @@ class HTTPClient(object): d.addCallback(readBody) return d - class ClientContextFactory(ClientContextFactory): - """ - A context factory that will verify the server's certificate against a - given CA certificate. - """ - - def __init__(self, cacert): - """ - Initialize the context factory. - - :param cacert: The CA certificate. - :type cacert: OpenSSL.crypto.X509 - """ - self._cacert = cacert - - def getContext(self, hostname, port): - opts = CertificateOptions(verify=True, caCerts=[self._cacert]) - return opts.getContext() - class StringBodyProducer(object): """ A producer that writes the body of a request to a consumer. -- cgit v1.2.3 From bf18c2bc6e3f533187281a3b31febd37ef22f8c0 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 27 May 2015 17:19:13 -0300 Subject: [feat] Make it optional to have a dedicated pool As @meskio pointed out, some cases could need a dedicated pool with different parameters. This is a suggested implementation where the pool is reused by default, creating a dedicated one just if needed/asked. This way we ensure that resources are under control and special cases are still handled. --- src/leap/common/http.py | 14 ++++++--- src/leap/common/tests/test_http.py | 62 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 src/leap/common/tests/test_http.py (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index 1dc5642..1e384e5 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -44,13 +44,21 @@ from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer +def createPool(maxPersistentPerHost=10, persistent=True): + pool = HTTPConnectionPool(reactor, persistent) + pool.maxPersistentPerHost = maxPersistentPerHost + return pool + +_pool = createPool() + + class HTTPClient(object): """ HTTP client done the twisted way, with a main focus on pinning the SSL certificate. """ - def __init__(self, cert_file=None): + def __init__(self, cert_file=None, pool=_pool): """ Init the HTTP client @@ -58,15 +66,13 @@ class HTTPClient(object): system's CAs will be used. :type cert_file: str """ - self._pool = HTTPConnectionPool(reactor, persistent=True) - self._pool.maxPersistentPerHost = 10 policy = get_compatible_ssl_context_factory(cert_file) self._agent = Agent( reactor, policy, - pool=self._pool) + pool=pool) def request(self, url, method='GET', body=None, headers={}): """ diff --git a/src/leap/common/tests/test_http.py b/src/leap/common/tests/test_http.py new file mode 100644 index 0000000..e240ca3 --- /dev/null +++ b/src/leap/common/tests/test_http.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# test_http.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for: + * leap/common/http.py +""" +import os +try: + import unittest2 as unittest +except ImportError: + import unittest + +from leap.common import http +from leap.common.testing.basetest import BaseLeapTest + +TEST_CERT_PEM = os.path.join( + os.path.split(__file__)[0], + '..', 'testing', "leaptest_combined_keycert.pem") + + +class HTTPClientTest(BaseLeapTest): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_agents_sharing_pool_by_default(self): + client = http.HTTPClient() + client2 = http.HTTPClient(TEST_CERT_PEM) + self.assertNotEquals(client._agent, client2._agent, "Expected dedicated agents") + self.assertEquals(client._agent._pool, client2._agent._pool, "Pool was not reused by default") + + def test_agent_can_have_dedicated_custom_pool(self): + custom_pool = http.createPool(maxPersistentPerHost=42, persistent=False) + self.assertEquals(custom_pool.maxPersistentPerHost, 42, + "Custom persistent connections limit is not being respected") + self.assertFalse(custom_pool.persistent, + "Custom persistence is not being respected") + default_client = http.HTTPClient() + custom_client = http.HTTPClient(pool=custom_pool) + + self.assertNotEquals(default_client._agent, custom_client._agent, "No agent reuse is expected") + self.assertEquals(custom_pool, custom_client._agent._pool, "Custom pool usage was not respected") + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3 From 76436726663971ebd58bf2c758b52abb10f7c242 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 2 Jun 2015 17:27:42 -0400 Subject: [bug] allow ipc socket types previous regex wasn't capturing addresses of type ipc:// Closes: #7089 --- src/leap/common/events/tests/__init__.py | 0 .../common/events/tests/test_zmq_components.py | 51 ++++++++++++++++++++++ src/leap/common/events/zmq_components.py | 24 +++++----- 3 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 src/leap/common/events/tests/__init__.py create mode 100644 src/leap/common/events/tests/test_zmq_components.py (limited to 'src/leap/common') diff --git a/src/leap/common/events/tests/__init__.py b/src/leap/common/events/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/leap/common/events/tests/test_zmq_components.py b/src/leap/common/events/tests/test_zmq_components.py new file mode 100644 index 0000000..c51e37e --- /dev/null +++ b/src/leap/common/events/tests/test_zmq_components.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# test_zmq_components.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for the zmq_components module. +""" +try: + import unittest2 as unittest +except ImportError: + import unittest + +from leap.common.events import zmq_components + + +class AddrParseTestCase(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_addr_parsing(self): + addr_re = zmq_components.ADDRESS_RE + + self.assertEqual( + addr_re.search("ipc:///tmp/foo/bar/baaz-2/foo.0").groups(), + ("ipc", "/tmp/foo/bar/baaz-2/foo.0", None)) + self.assertEqual( + addr_re.search("tcp://localhost:9000").groups(), + ("tcp", "localhost", "9000")) + self.assertEqual( + addr_re.search("tcp://127.0.0.1:9000").groups(), + ("tcp", "127.0.0.1", "9000")) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/leap/common/events/zmq_components.py b/src/leap/common/events/zmq_components.py index 4fb95d3..3b88862 100644 --- a/src/leap/common/events/zmq_components.py +++ b/src/leap/common/events/zmq_components.py @@ -45,7 +45,7 @@ from leap.common.zmq_utils import PUBLIC_KEYS_PREFIX logger = logging.getLogger(__name__) -ADDRESS_RE = re.compile("(.+)://(.+):([0-9]+)") +ADDRESS_RE = re.compile("^([a-z]+)://([^:]+):?(\d+)?$") class TxZmqComponent(object): @@ -63,7 +63,7 @@ class TxZmqComponent(object): """ self._factory = txzmq.ZmqFactory() self._factory.registerForShutdown() - if path_prefix == None: + if path_prefix is None: path_prefix = get_path_prefix() self._config_prefix = os.path.join(path_prefix, "leap", "events") self._connections = [] @@ -125,15 +125,19 @@ class TxZmqComponent(object): socket.curve_publickey = public socket.curve_secretkey = secret self._start_thread_auth(connection.socket) - # check if port was given - protocol, addr, port = ADDRESS_RE.match(address).groups() - if port == "0": - port = socket.bind_to_random_port("%s://%s" % (protocol, addr)) + + proto, addr, port = ADDRESS_RE.search(address).groups() + + if port is None: + params = proto, addr + port = socket.bind("%s://%s" % params) + # XXX this log doesn't appear + logger.debug("Binded %s to %s://%s." % ((connClass,) + params)) else: - socket.bind(address) - port = int(port) - logger.debug("Binded %s to %s://%s:%d." - % (connClass, protocol, addr, port)) + params = proto, addr, int(port) + socket.bind("%s://%s:%d" % params) + # XXX this log doesn't appear + logger.debug("Binded %s to %s://%s:%d." % ((connClass,) + params)) self._connections.append(connection) return connection, port -- cgit v1.2.3 From b67648aa666345b0800b48f6c203538b21c9a201 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 8 Jun 2015 15:18:00 -0300 Subject: [bug] Makes request method respect a hard limit Altough we specify maxPersistentPerHost, Twisted won't stop opening connections after that. This limit is used just to keep the size of persistent connections pool under control. Additional connections will be made as non persistent. So, if we ask 10000 requests, it will open 10000 connections immediately and leave 10 open after all finished. For checking this behavior, see getConnection from Twisted source: http://twistedmatrix.com/trac/browser/tags/releases/twisted-15.2.1/twisted/web/client.py#L1203 I tested this by using http_target from soledad without a local database to download all encrypted docs from one account with 1700 of them. The program just hangs and crashes with 1000+ connections and "Too many files open" warnings. With this fix, it was able to download normally, respecting the maxPersistentPerHost as a limiter. :) --- src/leap/common/http.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index 1e384e5..d4a214c 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -35,6 +35,7 @@ from leap.common.certs import get_compatible_ssl_context_factory from zope.interface import implements from twisted.internet import reactor +from twisted.internet import defer from twisted.internet.defer import succeed from twisted.web.client import Agent @@ -56,6 +57,13 @@ class HTTPClient(object): """ HTTP client done the twisted way, with a main focus on pinning the SSL certificate. + + By default, it uses a shared connection pool. If you want a dedicated + one, create and pass on __init__ pool parameter. + Please note that this client will limit the maximum amount of connections + by using a DeferredSemaphore. + This limit is equal to the maxPersistentPerHost used on pool and is needed + in order to avoid resource abuse on huge requests batches. """ def __init__(self, cert_file=None, pool=_pool): @@ -65,6 +73,9 @@ class HTTPClient(object): :param cert_file: The path to the certificate file, if None given the system's CAs will be used. :type cert_file: str + :param pool: An optional dedicated connection pool to override the + default shared one. + :type pool: HTTPConnectionPool """ policy = get_compatible_ssl_context_factory(cert_file) @@ -73,6 +84,7 @@ class HTTPClient(object): reactor, policy, pool=pool) + self._semaphore = defer.DeferredSemaphore(pool.maxPersistentPerHost) def request(self, url, method='GET', body=None, headers={}): """ @@ -92,8 +104,9 @@ class HTTPClient(object): """ if body: body = HTTPClient.StringBodyProducer(body) - d = self._agent.request( - method, url, headers=Headers(headers), bodyProducer=body) + d = self._semaphore.run(self._agent.request, + method, url, headers=Headers(headers), + bodyProducer=body) d.addCallback(readBody) return d -- cgit v1.2.3 From b33f0ef3485311f87990409ef8ba2bcb8b26dc5d Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 11 Jun 2015 11:54:10 -0300 Subject: [bug] remove extraneous data from events logs The emission of an event was being logged twice, and the second time was logging the pickled content of the event. This pickled content contained line breaks and other things that caused strange output on the client log. This commit removes the second loggin of the event pickled content. Closes #7130. --- src/leap/common/events/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index 0706fe3..4852b5a 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -173,7 +173,7 @@ class EventsClient(object): :param content: The content of the event. :type content: list """ - logger.debug("Sending event: (%s, %s)" % (event, content)) + logger.debug("Emitting event: (%s, %s)" % (event, content)) self._send(str(event) + b'\0' + pickle.dumps(content)) def _handle_event(self, event, content): @@ -368,7 +368,6 @@ class EventsClientThread(threading.Thread, EventsClient): :param data: The data to be sent. :type event: str """ - logger.debug("Sending data: %s" % data) # add send() as a callback for ioloop so it works between threads self._loop.add_callback(lambda: self._push.send(data)) -- cgit v1.2.3 From e91c6b2daf15d849de5a5fd72436f9f88505250e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 5 Jun 2015 13:11:56 -0400 Subject: [feature] utility collect_plugins to be used in post-sync hooks. Related: #6996 Releases: 0.4.1 --- src/leap/common/plugins.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/leap/common/plugins.py (limited to 'src/leap/common') diff --git a/src/leap/common/plugins.py b/src/leap/common/plugins.py new file mode 100644 index 0000000..bf0cd48 --- /dev/null +++ b/src/leap/common/plugins.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# plugins.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Twisted plugins leap utilities. +""" +import os.path + +from twisted.plugin import getPlugins + +from leap.common.config import get_path_prefix + +# A whitelist of modules from where to collect plugins dynamically. +# For the moment restricted to leap namespace, but the idea is that we can pass +# other "trusted" modules as options to the initialization of soledad. + +# TODO discover all the namespace automagically + +PLUGGABLE_LEAP_MODULES = ('mail', 'keymanager') + +_preffix = get_path_prefix() +rc_file = os.path.join(_preffix, "leap", "leap.cfg") + + +def _get_extra_pluggable_modules(): + import ConfigParser + config = ConfigParser.RawConfigParser() + config.read(rc_file) + try: + modules = eval( + config.get('plugins', 'extra_pluggable_modules'), {}, {}) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + modules = [] + return modules + +if os.path.isfile(rc_file): + # TODO in the case of being called from the standalone client, + # we should pass the flag in some other way. + EXTRA_PLUGGABLE_MODULES = _get_extra_pluggable_modules() +else: + EXTRA_PLUGGABLE_MODULES = [] + + +def collect_plugins(interface): + """ + Traverse a whitelist of modules and collect all the plugins that implement + the passed interface. + """ + plugins = [] + for namespace in PLUGGABLE_LEAP_MODULES: + try: + module = __import__('leap.%s.plugins' % namespace, fromlist='.') + plugins = plugins + list(getPlugins(interface, module)) + except ImportError: + pass + for namespace in EXTRA_PLUGGABLE_MODULES: + try: + module = __import__('%s.plugins' % namespace, fromlist='.') + plugins = plugins + list(getPlugins(interface, module)) + except ImportError: + pass + return plugins -- cgit v1.2.3 From c9f07ff0041fac8b5124a15a8053bb9cc9f03bb2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 10 Jun 2015 14:52:11 -0400 Subject: [bug] catch missing section header error --- src/leap/common/plugins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/leap/common') diff --git a/src/leap/common/plugins.py b/src/leap/common/plugins.py index bf0cd48..04152f9 100644 --- a/src/leap/common/plugins.py +++ b/src/leap/common/plugins.py @@ -42,7 +42,8 @@ def _get_extra_pluggable_modules(): try: modules = eval( config.get('plugins', 'extra_pluggable_modules'), {}, {}) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, + ConfigParser.MissingSectionHeaderError): modules = [] return modules -- cgit v1.2.3 From c6107b88ba1eaf7e7ca97d0444e7444634aa98c2 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 26 Jun 2015 14:44:37 -0300 Subject: [bug] allow passing ':0' as port in events address --- src/leap/common/events/zmq_components.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/zmq_components.py b/src/leap/common/events/zmq_components.py index 3b88862..04f71e0 100644 --- a/src/leap/common/events/zmq_components.py +++ b/src/leap/common/events/zmq_components.py @@ -128,9 +128,9 @@ class TxZmqComponent(object): proto, addr, port = ADDRESS_RE.search(address).groups() - if port is None: + if port is None or port is '0': params = proto, addr - port = socket.bind("%s://%s" % params) + port = socket.bind_to_random_port("%s://%s" % params) # XXX this log doesn't appear logger.debug("Binded %s to %s://%s." % ((connClass,) + params)) else: -- cgit v1.2.3 From 10d7ebdf5226b74c54850d205a807cc650843efe Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 26 Jun 2015 14:45:19 -0300 Subject: [bug] run callback from thread in events client --- src/leap/common/events/client.py | 36 +++++++++++++++++++++++++++++++++--- src/leap/common/events/txclient.py | 13 +++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index 4852b5a..1a026ea 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -180,14 +180,30 @@ class EventsClient(object): """ Handle an incoming event. - :param msg: The incoming message. - :type msg: list(str) + :param event: The event to be sent. + :type event: Event + :param content: The content of the event. + :type content: list """ logger.debug("Handling event %s..." % event) for uid in self._callbacks[event].keys(): callback = self._callbacks[event][uid] logger.debug("Executing callback %s." % uid) - callback(event, *content) + self._run_callback(callback, event, content) + + @abstractmethod + def _run_callback(self, callback, event, content): + """ + Run a callback. + + :param callback: The callback to be run. + :type callback: callable(event, *content) + :param event: The event to be sent. + :type event: Event + :param content: The content of the event. + :type content: list + """ + pass @abstractmethod def _subscribe(self, tag): @@ -371,6 +387,20 @@ class EventsClientThread(threading.Thread, EventsClient): # add send() as a callback for ioloop so it works between threads self._loop.add_callback(lambda: self._push.send(data)) + def _run_callback(self, callback, event, content): + """ + Run a callback. + + :param callback: The callback to be run. + :type callback: callable(event, *content) + :param event: The event to be sent. + :type event: Event + :param content: The content of the event. + :type content: list + """ + from twisted.internet import reactor + reactor.callFromThread(callback, event, *content) + def register(self, event, callback, uid=None, replace=False): """ Register a callback to be executed when an event is received. diff --git a/src/leap/common/events/txclient.py b/src/leap/common/events/txclient.py index 8206ed5..0dcfc08 100644 --- a/src/leap/common/events/txclient.py +++ b/src/leap/common/events/txclient.py @@ -112,6 +112,19 @@ class EventsTxClient(TxZmqClientComponent, EventsClient): """ self._push.send(data) + def _run_callback(self, callback, event, content): + """ + Run a callback. + + :param callback: The callback to be run. + :type callback: callable(event, *content) + :param event: The event to be sent. + :type event: Event + :param content: The content of the event. + :type content: list + """ + callback(event, *content) + def shutdown(self): TxZmqClientComponent.shutdown(self) EventsClient.shutdown(self) -- cgit v1.2.3 From f508d3629686d1ad4574e5a7ec67f8b3283f7c5f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 20 Jul 2015 15:18:57 -0400 Subject: [refactor] remove use of reactor in threaded version of events client the idea is that we'll be able to use the threaded version of the client, which makes use of the tornado ioloop, in a non-twisted module, like the main graphical client probably will be in the near future. --- src/leap/common/events/client.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index 1a026ea..5ee617e 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -174,7 +174,8 @@ class EventsClient(object): :type content: list """ logger.debug("Emitting event: (%s, %s)" % (event, content)) - self._send(str(event) + b'\0' + pickle.dumps(content)) + payload = str(event) + b'\0' + pickle.dumps(content) + self._send(payload) def _handle_event(self, event, content): """ @@ -186,7 +187,7 @@ class EventsClient(object): :type content: list """ logger.debug("Handling event %s..." % event) - for uid in self._callbacks[event].keys(): + for uid in self._callbacks[event]: callback = self._callbacks[event][uid] logger.debug("Executing callback %s." % uid) self._run_callback(callback, event, content) @@ -398,8 +399,7 @@ class EventsClientThread(threading.Thread, EventsClient): :param content: The content of the event. :type content: list """ - from twisted.internet import reactor - reactor.callFromThread(callback, event, *content) + self._loop.add_callback(lambda: callback(event, *content)) def register(self, event, callback, uid=None, replace=False): """ @@ -422,7 +422,8 @@ class EventsClientThread(threading.Thread, EventsClient): callback identified by the given uid and replace is False. """ self.ensure_client() - return EventsClient.register(self, event, callback, uid=uid, replace=replace) + return EventsClient.register( + self, event, callback, uid=uid, replace=replace) def unregister(self, event, uid=None): """ -- cgit v1.2.3 From 87339921eac261954e39901e3563851830309cc5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 20 Jul 2015 19:18:03 -0400 Subject: [bug] do not add a port string to non-tcp addresses this, together with the events server registration, was breaking the events signalling on the client whenever it used ipc:// sockets. --- src/leap/common/events/zmq_components.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/zmq_components.py b/src/leap/common/events/zmq_components.py index 04f71e0..f99c754 100644 --- a/src/leap/common/events/zmq_components.py +++ b/src/leap/common/events/zmq_components.py @@ -128,16 +128,21 @@ class TxZmqComponent(object): proto, addr, port = ADDRESS_RE.search(address).groups() - if port is None or port is '0': - params = proto, addr - port = socket.bind_to_random_port("%s://%s" % params) - # XXX this log doesn't appear - logger.debug("Binded %s to %s://%s." % ((connClass,) + params)) + if proto == "tcp": + if port is None or port is '0': + params = proto, addr + port = socket.bind_to_random_port("%s://%s" % params) + logger.debug("Binded %s to %s://%s." % ((connClass,) + params)) + else: + params = proto, addr, int(port) + socket.bind("%s://%s:%d" % params) + logger.debug( + "Binded %s to %s://%s:%d." % ((connClass,) + params)) else: - params = proto, addr, int(port) - socket.bind("%s://%s:%d" % params) - # XXX this log doesn't appear - logger.debug("Binded %s to %s://%s:%d." % ((connClass,) + params)) + params = proto, addr + socket.bind("%s://%s" % params) + logger.debug( + "Binded %s to %s://%s" % ((connClass,) + params)) self._connections.append(connection) return connection, port -- cgit v1.2.3 From 467b14fa2e29ecd6f41d4834b00593d8c86cddc5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 13 Jul 2015 11:48:47 -0400 Subject: [feature] add global flag for disabling the events framework this will be used to allow the unittests to disable the events framework. this way, emit() will become a passthrough. note that, until now, the basetest class is making use of the threaded version of the client, which launches a zmq tornado-based ioloop. this is wrong, and will have to be addressed in a future commit. we'll have to make use of the global EVENTS_ENABLED flag in the txclient version when those changes are made. Related: #7259 Relases: 0.4.2 --- src/leap/common/events/__init__.py | 7 +++---- src/leap/common/events/client.py | 12 +++++------- src/leap/common/events/flags.py | 28 ++++++++++++++++++++++++++++ src/leap/common/events/txclient.py | 6 +----- src/leap/common/testing/basetest.py | 21 +++++++++++++-------- 5 files changed, 50 insertions(+), 24 deletions(-) create mode 100644 src/leap/common/events/flags.py (limited to 'src/leap/common') diff --git a/src/leap/common/events/__init__.py b/src/leap/common/events/__init__.py index 9269b9a..87ed8ae 100644 --- a/src/leap/common/events/__init__.py +++ b/src/leap/common/events/__init__.py @@ -14,8 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ This is an events mechanism that uses a server to allow for emitting events between clients. @@ -37,13 +35,13 @@ To emit an event, use leap.common.events.emit(): >>> from leap.common.events import catalog >>> emit(catalog.CLIENT_UID) """ - - import logging import argparse from leap.common.events import client from leap.common.events import server +from leap.common.events.flags import set_events_enabled + from leap.common.events import catalog @@ -52,6 +50,7 @@ __all__ = [ "unregister", "emit", "catalog", + "set_events_enabled" ] diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index 5ee617e..1744341 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -14,8 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ The client end point of the events mechanism. @@ -27,8 +25,6 @@ When a client registers a callback for a given event, it also tells the server that it wants to be notified whenever events of that type are sent by some other client. """ - - import logging import collections import uuid @@ -59,6 +55,7 @@ from leap.common.zmq_utils import PUBLIC_KEYS_PREFIX from leap.common.events.errors import CallbackAlreadyRegisteredError from leap.common.events.server import EMIT_ADDR from leap.common.events.server import REG_ADDR +from leap.common.events import flags from leap.common.events import catalog @@ -173,9 +170,10 @@ class EventsClient(object): :param content: The content of the event. :type content: list """ - logger.debug("Emitting event: (%s, %s)" % (event, content)) - payload = str(event) + b'\0' + pickle.dumps(content) - self._send(payload) + if flags.EVENTS_ENABLED: + logger.debug("Emitting event: (%s, %s)" % (event, content)) + payload = str(event) + b'\0' + pickle.dumps(content) + self._send(payload) def _handle_event(self, event, content): """ diff --git a/src/leap/common/events/flags.py b/src/leap/common/events/flags.py new file mode 100644 index 0000000..137f663 --- /dev/null +++ b/src/leap/common/events/flags.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# __init__.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Flags for the events framework. +""" +from leap.common.check import leap_assert + +EVENTS_ENABLED = True + + +def set_events_enabled(flag): + leap_assert(isinstance(flag, bool)) + global EVENTS_ENABLED + EVENTS_ENABLED = flag diff --git a/src/leap/common/events/txclient.py b/src/leap/common/events/txclient.py index 0dcfc08..dfd0533 100644 --- a/src/leap/common/events/txclient.py +++ b/src/leap/common/events/txclient.py @@ -14,8 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ The client end point of the events mechanism, implemented using txzmq. @@ -27,8 +25,6 @@ When a client registers a callback for a given event, it also tells the server that it wants to be notified whenever events of that type are sent by some other client. """ - - import logging import pickle @@ -62,7 +58,7 @@ class EventsTxClient(TxZmqClientComponent, EventsClient): """ def __init__(self, emit_addr=EMIT_ADDR, reg_addr=REG_ADDR, - path_prefix=None): + path_prefix=None): """ Initialize the events server. """ diff --git a/src/leap/common/testing/basetest.py b/src/leap/common/testing/basetest.py index 3fbcf33..3d3cee0 100644 --- a/src/leap/common/testing/basetest.py +++ b/src/leap/common/testing/basetest.py @@ -30,8 +30,11 @@ except ImportError: from leap.common.check import leap_assert from leap.common.events import server as events_server from leap.common.events import client as events_client +from leap.common.events import flags, set_events_enabled from leap.common.files import mkdir_p, check_and_fix_urw_only +set_events_enabled(False) + class BaseLeapTest(unittest.TestCase): """ @@ -73,12 +76,13 @@ class BaseLeapTest(unittest.TestCase): @classmethod def _init_events(cls): - cls._server = events_server.ensure_server( - emit_addr="tcp://127.0.0.1:0", - reg_addr="tcp://127.0.0.1:0") - events_client.configure_client( - emit_addr="tcp://127.0.0.1:%d" % cls._server.pull_port, - reg_addr="tcp://127.0.0.1:%d" % cls._server.pub_port) + if flags.EVENTS_ENABLED: + cls._server = events_server.ensure_server( + emit_addr="tcp://127.0.0.1:0", + reg_addr="tcp://127.0.0.1:0") + events_client.configure_client( + emit_addr="tcp://127.0.0.1:%d" % cls._server.pull_port, + reg_addr="tcp://127.0.0.1:%d" % cls._server.pub_port) @classmethod def tearDownEnv(cls): @@ -87,8 +91,9 @@ class BaseLeapTest(unittest.TestCase): - restores the default PATH and HOME variables - removes the temporal folder """ - events_client.shutdown() - cls._server.shutdown() + if flags.EVENTS_ENABLED: + events_client.shutdown() + cls._server.shutdown() os.environ["PATH"] = cls.old_path os.environ["HOME"] = cls.old_home -- cgit v1.2.3 From 46870f1e26ef159bf6fe4744aba3b00a5e81cc3b Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 8 Jul 2015 19:14:00 -0300 Subject: [feat] add close method for http agent The ability to close cached connections is needed in order to have a clean reactor when the program ends. --- src/leap/common/http.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index d4a214c..8d22f2c 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -80,6 +80,7 @@ class HTTPClient(object): policy = get_compatible_ssl_context_factory(cert_file) + self._pool = pool self._agent = Agent( reactor, policy, @@ -110,6 +111,12 @@ class HTTPClient(object): d.addCallback(readBody) return d + def close(self): + """ + Close any cached connections. + """ + self._pool.closeCachedConnections() + class StringBodyProducer(object): """ A producer that writes the body of a request to a consumer. -- cgit v1.2.3 From 34f6b07e08540fd9b2ec473d1e4e5a15be4feacc Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 8 Jul 2015 19:15:56 -0300 Subject: [bug] add http request timeout The connectTimeout parameter of twisted.web.client.Agent only acts on the connection setup, and the Agent will wait forever for incoming data after the connection has been established. This commit adds a timeout for the connection, and will cancel the deferred if the result has not been received after a certain number of seconds. --- src/leap/common/http.py | 260 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 214 insertions(+), 46 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index 8d22f2c..c93e65b 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -36,21 +36,24 @@ from zope.interface import implements from twisted.internet import reactor from twisted.internet import defer -from twisted.internet.defer import succeed +from twisted.python import failure from twisted.web.client import Agent from twisted.web.client import HTTPConnectionPool +from twisted.web.client import _HTTP11ClientFactory as HTTP11ClientFactory from twisted.web.client import readBody from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer +from twisted.web._newclient import HTTP11ClientProtocol -def createPool(maxPersistentPerHost=10, persistent=True): - pool = HTTPConnectionPool(reactor, persistent) - pool.maxPersistentPerHost = maxPersistentPerHost - return pool +__all__ = ["HTTPClient"] -_pool = createPool() + +# A default HTTP timeout is used for 2 distinct purposes: +# 1. as HTTP connection timeout, prior to connection estabilshment. +# 2. as data reception timeout, after the connection has been established. +DEFAULT_HTTP_TIMEOUT = 30 # seconds class HTTPClient(object): @@ -66,28 +69,36 @@ class HTTPClient(object): in order to avoid resource abuse on huge requests batches. """ - def __init__(self, cert_file=None, pool=_pool): + def __init__(self, cert_file=None, timeout=DEFAULT_HTTP_TIMEOUT): """ Init the HTTP client :param cert_file: The path to the certificate file, if None given the system's CAs will be used. :type cert_file: str - :param pool: An optional dedicated connection pool to override the - default shared one. - :type pool: HTTPConnectionPool + :param timeout: The amount of time that this Agent will wait for the + peer to accept a connection and for each request to be + finished. If a pool is passed, then this argument is + ignored. + :type timeout: float """ - policy = get_compatible_ssl_context_factory(cert_file) - - self._pool = pool + self._timeout = timeout + self._pool = self._createPool() self._agent = Agent( reactor, - policy, - pool=pool) - self._semaphore = defer.DeferredSemaphore(pool.maxPersistentPerHost) + get_compatible_ssl_context_factory(cert_file), + pool=self._pool, + connectTimeout=self._timeout) + self._semaphore = defer.DeferredSemaphore( + self._pool.maxPersistentPerHost) - def request(self, url, method='GET', body=None, headers={}): + def _createPool(self, maxPersistentPerHost=10, persistent=True): + pool = _HTTPConnectionPool(reactor, persistent, self._timeout) + pool.maxPersistentPerHost = maxPersistentPerHost + return pool + + def _request(self, url, method, body, headers): """ Perform an HTTP request. @@ -104,51 +115,208 @@ class HTTPClient(object): :rtype: twisted.internet.defer.Deferred """ if body: - body = HTTPClient.StringBodyProducer(body) - d = self._semaphore.run(self._agent.request, - method, url, headers=Headers(headers), - bodyProducer=body) + body = _StringBodyProducer(body) + d = self._agent.request( + method, url, headers=Headers(headers), bodyProducer=body) d.addCallback(readBody) return d + def request(self, url, method='GET', body=None, headers={}): + """ + Perform an HTTP request, but limit the maximum amount of concurrent + connections. + + :param url: The URL for the request. + :type url: str + :param method: The HTTP method of the request. + :type method: str + :param body: The body of the request, if any. + :type body: str + :param headers: The headers of the request. + :type headers: dict + + :return: A deferred that fires with the body of the request. + :rtype: twisted.internet.defer.Deferred + """ + return self._semaphore.run(self._request, url, method, body, headers) + def close(self): """ Close any cached connections. """ self._pool.closeCachedConnections() - class StringBodyProducer(object): +# +# An IBodyProducer to write the body of an HTTP request as a string. +# + +class _StringBodyProducer(object): + """ + A producer that writes the body of a request to a consumer. + """ + + implements(IBodyProducer) + + def __init__(self, body): + """ + Initialize the string produer. + + :param body: The body of the request. + :type body: str + """ + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + """ + Write the body to the consumer. + + :param consumer: Any IConsumer provider. + :type consumer: twisted.internet.interfaces.IConsumer + + :return: A successful deferred. + :rtype: twisted.internet.defer.Deferred + """ + consumer.write(self.body) + return defer.succeed(None) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + + +# +# Patched twisted.web classes +# + +class _HTTP11ClientProtocol(HTTP11ClientProtocol): + """ + A timeout-able HTTP 1.1 client protocol, that is instantiated by the + _HTTP11ClientFactory below. + """ + + def __init__(self, quiescentCallback, timeout): + """ + Initialize the protocol. + + :param quiescentCallback: + :type quiescentCallback: callable + :param timeout: A timeout, in seconds, for requests made by this + protocol. + :type timeout: float + """ + HTTP11ClientProtocol.__init__(self, quiescentCallback) + self._timeout = timeout + self._timeoutCall = None + + def request(self, request): + """ + Issue request over self.transport and return a Deferred which + will fire with a Response instance or an error. + + :param request: The object defining the parameters of the request to + issue. + :type request: twisted.web._newclient.Request + + :return: A deferred which fires after the request has finished. + :rtype: Deferred + """ + d = HTTP11ClientProtocol.request(self, request) + if self._timeout: + self._last_buffer_len = 0 + timeoutCall = reactor.callLater( + self._timeout, self._doTimeout, request) + self._timeoutCall = timeoutCall + return d + + def _doTimeout(self, request): """ - A producer that writes the body of a request to a consumer. + Give up the request because of a timeout. + + :param request: The object defining the parameters of the request to + issue. + :type request: twisted.web._newclient.Request + """ + self._giveUp( + failure.Failure( + defer.TimeoutError( + "Getting %s took longer than %s seconds." + % (request.absoluteURI, self._timeout)))) + + def _cancelTimeout(self): """ + Cancel the request timeout, when it's finished. + """ + if self._timeoutCall.active(): + self._timeoutCall.cancel() + self._timeoutCall = None + + def _finishResponse_WAITING(self, rest): + """ + Cancel the timeout when finished receiving the response. + """ + self._cancelTimeout() + HTTP11ClientProtocol._finishResponse_WAITING(self, rest) - implements(IBodyProducer) + def _finishResponse_TRANSMITTING(self, rest): + """ + Cancel the timeout when finished receiving the response. + """ + self._cancelTimeout() + HTTP11ClientProtocol._finishResponse_TRANSMITTING(self, rest) - def __init__(self, body): - """ - Initialize the string produer. + def dataReceived(self, bytes): + """ + Receive some data and extend the timeout period of this request. + + :param bytes: A string of indeterminate length. + :type bytes: str + """ + HTTP11ClientProtocol.dataReceived(self, bytes) + if self._timeoutCall and self._timeoutCall.active(): + self._timeoutCall.reset(self._timeout) - :param body: The body of the request. - :type body: str - """ - self.body = body - self.length = len(body) - def startProducing(self, consumer): - """ - Write the body to the consumer. +class _HTTP11ClientFactory(HTTP11ClientFactory): + """ + A timeout-able HTTP 1.1 client protocol factory. + """ - :param consumer: Any IConsumer provider. - :type consumer: twisted.internet.interfaces.IConsumer + def __init__(self, quiescentCallback, timeout): + """ + :param quiescentCallback: The quiescent callback to be passed to + protocol instances, used to return them to + the connection pool. + :type quiescentCallback: callable(Protocol) + :param timeout: The timeout, in seconds, for requests made by + protocols created by this factory. + :type timeout: float + """ + HTTP11ClientFactory.__init__(self, quiescentCallback) + self._timeout = timeout + + def buildProtocol(self, _): + """ + Build the HTTP 1.1 client protocol. + """ + return _HTTP11ClientProtocol(self._quiescentCallback, self._timeout) + + +class _HTTPConnectionPool(HTTPConnectionPool): + """ + A timeout-able HTTP connection pool. + """ - :return: A successful deferred. - :rtype: twisted.internet.defer.Deferred - """ - consumer.write(self.body) - return succeed(None) + _factory = _HTTP11ClientFactory - def pauseProducing(self): - pass + def __init__(self, reactor, persistent, timeout): + HTTPConnectionPool.__init__(self, reactor, persistent=persistent) + self._timeout = timeout - def stopProducing(self): - pass + def _newConnection(self, key, endpoint): + def quiescentCallback(protocol): + self._putConnection(key, protocol) + factory = self._factory(quiescentCallback, timeout=self._timeout) + return endpoint.connect(factory) -- cgit v1.2.3 From 07d421a32f3bb45932668f4951233166ada4e770 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 23 Jul 2015 17:03:50 -0300 Subject: [bug] fix events ioloop client tests The events ioloop client is not completelly thread safe, so we have to use reactor.callFromThread whenever we need it to be thread-safe. Examples of this behaviour are the events ioloop client tests that depend on the firing of deferreds, which are not thread safe. This commit fixes tese tests. --- src/leap/common/tests/test_events.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/tests/test_events.py b/src/leap/common/tests/test_events.py index 7ef3e1b..611781c 100644 --- a/src/leap/common/tests/test_events.py +++ b/src/leap/common/tests/test_events.py @@ -20,6 +20,7 @@ import os import logging import time +from twisted.internet.reactor import callFromThread from twisted.trial import unittest from twisted.internet import defer @@ -80,8 +81,8 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d = defer.Deferred() - cbk_fail = lambda event, _: d.errback(event) - cbk_succeed = lambda event, _: d.callback(event) + cbk_fail = lambda event, _: callFromThread(d.errback, event) + cbk_succeed = lambda event, _: callFromThread(d.callback, event) self._client.register(event, cbk_fail, uid=1) self._client.register(event, cbk_succeed, uid=1, replace=True) self._client.emit(event, None) @@ -105,9 +106,9 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d1 = defer.Deferred() - cbk1 = lambda event, _: d1.callback(event) + cbk1 = lambda event, _: callFromThread(d1.callback, event) d2 = defer.Deferred() - cbk2 = lambda event, _: d2.callback(event) + cbk2 = lambda event, _: callFromThread(d2.callback, event) self._client.register(event, cbk1) self._client.register(event, cbk2) self._client.emit(event, None) @@ -121,7 +122,7 @@ class EventsGenericClientTestCase(object): event = catalog.CLIENT_UID d = defer.Deferred() def cbk(events, _): - d.callback(event) + callFromThread(d.callback, event) self._client.register(event, cbk) self._client.emit(event, None) return d @@ -133,14 +134,17 @@ class EventsGenericClientTestCase(object): event1 = catalog.CLIENT_UID d = defer.Deferred() # register more than one callback for the same event - self._client.register(event1, lambda ev, _: d.errback(None)) - self._client.register(event1, lambda ev, _: d.errback(None)) + self._client.register( + event1, lambda ev, _: callFromThread(d.errback, None)) + self._client.register( + event1, lambda ev, _: callFromThread(d.errback, None)) # unregister and emit the event self._client.unregister(event1) self._client.emit(event1, None) # register and emit another event so the deferred can succeed event2 = catalog.CLIENT_SESSION_ID - self._client.register(event2, lambda ev, _: d.callback(None)) + self._client.register( + event2, lambda ev, _: callFromThread(d.callback, None)) self._client.emit(event2, None) return d @@ -151,9 +155,11 @@ class EventsGenericClientTestCase(object): event = catalog.CLIENT_UID d = defer.Deferred() # register one callback that would fail - uid = self._client.register(event, lambda ev, _: d.errback(None)) + uid = self._client.register( + event, lambda ev, _: callFromThread(d.errback, None)) # register one callback that will succeed - self._client.register(event, lambda ev, _: d.callback(None)) + self._client.register( + event, lambda ev, _: callFromThread(d.callback, None)) # unregister by uid and emit the event self._client.unregister(event, uid=uid) self._client.emit(event, None) -- cgit v1.2.3 From 25d3b7c80f85cd94159b574274108061a94f1bc9 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Wed, 22 Jul 2015 15:38:56 -0300 Subject: [style] Fixed pep8 warnings --- src/leap/common/__init__.py | 4 +-- src/leap/common/_version.py | 40 ++++++++++++------------- src/leap/common/ca_bundle.py | 1 + src/leap/common/config/pluggableconfig.py | 27 ++++++++++------- src/leap/common/config/tests/test_baseconfig.py | 30 +++++++++---------- src/leap/common/http.py | 21 ++++++------- src/leap/common/tests/test_certs.py | 6 ++-- src/leap/common/tests/test_events.py | 33 +++++++++++++++----- src/leap/common/zmq_utils.py | 2 -- 9 files changed, 96 insertions(+), 68 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/__init__.py b/src/leap/common/__init__.py index 5619900..383e198 100644 --- a/src/leap/common/__init__.py +++ b/src/leap/common/__init__.py @@ -4,6 +4,7 @@ from leap.common import certs from leap.common import check from leap.common import files from leap.common import events +from ._version import get_versions logger = logging.getLogger(__name__) @@ -11,11 +12,10 @@ try: import pygeoip HAS_GEOIP = True except ImportError: - #logger.debug('PyGeoIP not found. Disabled Geo support.') + # logger.debug('PyGeoIP not found. Disabled Geo support.') HAS_GEOIP = False __all__ = ["certs", "check", "files", "events"] -from ._version import get_versions __version__ = get_versions()['version'] del get_versions diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index 597e2e4..de94ba8 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -1,5 +1,3 @@ - -IN_LONG_VERSION_PY = True # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (build by setup.py sdist) and build @@ -10,12 +8,16 @@ IN_LONG_VERSION_PY = True # versioneer-0.7+ (https://github.com/warner/python-versioneer) # these strings will be replaced by git during git-archive -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" - import subprocess import sys +import re +import os.path + +IN_LONG_VERSION_PY = True +git_refnames = "$Format:%d$" +git_full = "$Format:%H$" + def run_command(args, cwd=None, verbose=False): try: @@ -37,10 +39,6 @@ def run_command(args, cwd=None, verbose=False): return stdout -import sys -import re -import os.path - def get_expanded_variables(versionfile_source): # the code embedded in _version.py can just fetch the value of these # variables. When used from setup.py, we don't want to import @@ -48,7 +46,7 @@ def get_expanded_variables(versionfile_source): # used from _version.py. variables = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_source, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) @@ -63,12 +61,13 @@ def get_expanded_variables(versionfile_source): pass return variables + def versions_from_expanded_variables(variables, tag_prefix, verbose=False): refnames = variables["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + return {} # unexpanded, so not in an unpacked git-archive tarball refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -84,7 +83,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) + print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -93,13 +92,14 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } + return {"version": r, + "full": variables["full"].strip()} # no suitable tags, so we use the full revision id if verbose: print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } + return {"version": variables["full"].strip(), + "full": variables["full"].strip()} + def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): # this runs 'git' from the root of the source tree. That either means @@ -116,7 +116,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): here = os.path.abspath(__file__) except NameError: # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct + return {} # not always correct # versionfile_source is the relative path from the top of the source tree # (where the .git directory might live) to this file. Invert this to find @@ -163,7 +163,7 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False) here = os.path.abspath(__file__) except NameError: # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + return {} # without __file__, we have no hope # versionfile_source is the relative path from the top of the source # tree to _version.py. Invert this to find the root from __file__. root = here @@ -189,8 +189,9 @@ tag_prefix = "" parentdir_prefix = "leap.common-" versionfile_source = "src/leap/common/_version.py" + def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } + variables = {"refnames": git_refnames, "full": git_full} ver = versions_from_expanded_variables(variables, tag_prefix, verbose) if not ver: ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) @@ -200,4 +201,3 @@ def get_versions(default={"version": "unknown", "full": ""}, verbose=False): if not ver: ver = default return ver - diff --git a/src/leap/common/ca_bundle.py b/src/leap/common/ca_bundle.py index d8c72a6..2c41d18 100644 --- a/src/leap/common/ca_bundle.py +++ b/src/leap/common/ca_bundle.py @@ -28,6 +28,7 @@ _system = platform.system() IS_MAC = _system == "Darwin" + def where(): """ Return the preferred certificate bundle. diff --git a/src/leap/common/config/pluggableconfig.py b/src/leap/common/config/pluggableconfig.py index 8535fa6..1a98427 100644 --- a/src/leap/common/config/pluggableconfig.py +++ b/src/leap/common/config/pluggableconfig.py @@ -27,7 +27,7 @@ import urlparse import jsonschema -#from leap.base.util.translations import LEAPTranslatable +# from leap.base.util.translations import LEAPTranslatable from leap.common.check import leap_assert @@ -163,8 +163,8 @@ class TranslatableType(object): return data # LEAPTranslatable(data) # needed? we already have an extended dict... - #def get_prep_value(self, data): - #return dict(data) + # def get_prep_value(self, data): + # return dict(data) class URIType(object): @@ -283,9 +283,13 @@ class PluggableConfig(object): except BaseException, e: raise TypeCastException( "Could not coerce %s, %s, " - "to format %s: %s" % (key, value, - _ftype.__class__.__name__, - e)) + "to format %s: %s" % ( + key, + value, + _ftype.__class__.__name__, + e + ) + ) return config @@ -303,9 +307,12 @@ class PluggableConfig(object): except BaseException, e: raise TypeCastException( "Could not serialize %s, %s, " - "by format %s: %s" % (key, value, - _ftype.__class__.__name__, - e)) + "by format %s: %s" % ( + key, + value, + _ftype.__class__.__name__, + e) + ) else: config[key] = value return config @@ -435,7 +442,7 @@ class PluggableConfig(object): content = self.deserialize(string) if not string and fromfile is not None: - #import ipdb;ipdb.set_trace() + # import ipdb;ipdb.set_trace() content = self.deserialize(fromfile=fromfile) if not content: diff --git a/src/leap/common/config/tests/test_baseconfig.py b/src/leap/common/config/tests/test_baseconfig.py index 8bdf4d0..e17e82d 100644 --- a/src/leap/common/config/tests/test_baseconfig.py +++ b/src/leap/common/config/tests/test_baseconfig.py @@ -29,21 +29,21 @@ from mock import Mock # reduced eipconfig sample config sample_config = { "gateways": [ - { - "capabilities": { - "adblock": False, - "transport": ["openvpn"], - "user_ips": False - }, - "host": "host.dev.example.org", - }, { - "capabilities": { - "adblock": False, - "transport": ["openvpn"], - "user_ips": False - }, - "host": "host2.dev.example.org", - } + { + "capabilities": { + "adblock": False, + "transport": ["openvpn"], + "user_ips": False + }, + "host": "host.dev.example.org", + }, { + "capabilities": { + "adblock": False, + "transport": ["openvpn"], + "user_ips": False + }, + "host": "host2.dev.example.org", + } ], "default_language": "en", "languages": [ diff --git a/src/leap/common/http.py b/src/leap/common/http.py index c93e65b..56938b4 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -150,6 +150,7 @@ class HTTPClient(object): # An IBodyProducer to write the body of an HTTP request as a string. # + class _StringBodyProducer(object): """ A producer that writes the body of a request to a consumer. @@ -254,18 +255,18 @@ class _HTTP11ClientProtocol(HTTP11ClientProtocol): self._timeoutCall = None def _finishResponse_WAITING(self, rest): - """ - Cancel the timeout when finished receiving the response. - """ - self._cancelTimeout() - HTTP11ClientProtocol._finishResponse_WAITING(self, rest) + """ + Cancel the timeout when finished receiving the response. + """ + self._cancelTimeout() + HTTP11ClientProtocol._finishResponse_WAITING(self, rest) def _finishResponse_TRANSMITTING(self, rest): - """ - Cancel the timeout when finished receiving the response. - """ - self._cancelTimeout() - HTTP11ClientProtocol._finishResponse_TRANSMITTING(self, rest) + """ + Cancel the timeout when finished receiving the response. + """ + self._cancelTimeout() + HTTP11ClientProtocol._finishResponse_TRANSMITTING(self, rest) def dataReceived(self, bytes): """ diff --git a/src/leap/common/tests/test_certs.py b/src/leap/common/tests/test_certs.py index 999071f..209e051 100644 --- a/src/leap/common/tests/test_certs.py +++ b/src/leap/common/tests/test_certs.py @@ -60,11 +60,13 @@ class CertsTest(BaseLeapTest): self.assertTrue(certs.should_redownload(cert_path)) def test_should_redownload_if_before(self): - new_now = lambda: time.struct_time(CERT_NOT_BEFORE) + def new_now(): + time.struct_time(CERT_NOT_BEFORE) self.assertTrue(certs.should_redownload(TEST_CERT_PEM, now=new_now)) def test_should_redownload_if_after(self): - new_now = lambda: time.struct_time(CERT_NOT_AFTER) + def new_now(): + time.struct_time(CERT_NOT_AFTER) self.assertTrue(certs.should_redownload(TEST_CERT_PEM, now=new_now)) def test_not_should_redownload(self): diff --git a/src/leap/common/tests/test_events.py b/src/leap/common/tests/test_events.py index 611781c..e2a918f 100644 --- a/src/leap/common/tests/test_events.py +++ b/src/leap/common/tests/test_events.py @@ -1,4 +1,4 @@ -## -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # test_events.py # Copyright (C) 2013 LEAP # @@ -60,7 +60,10 @@ class EventsGenericClientTestCase(object): 'There should be no callback for this event.') # register one event event1 = catalog.CLIENT_UID - cbk1 = lambda event, _: True + + def cbk1(event, _): + return True + uid1 = self._client.register(event1, cbk1) # assert for correct registration self.assertTrue(len(callbacks) == 1) @@ -68,7 +71,10 @@ class EventsGenericClientTestCase(object): 'Could not register event in local client.') # register another event event2 = catalog.CLIENT_SESSION_ID - cbk2 = lambda event, _: True + + def cbk2(event, _): + return True + uid2 = self._client.register(event2, cbk2) # assert for correct registration self.assertTrue(len(callbacks) == 2) @@ -81,8 +87,13 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d = defer.Deferred() - cbk_fail = lambda event, _: callFromThread(d.errback, event) - cbk_succeed = lambda event, _: callFromThread(d.callback, event) + + def cbk_fail(event, _): + return callFromThread(d.errback, event) + + def cbk_succeed(event, _): + return callFromThread(d.callback, event) + self._client.register(event, cbk_fail, uid=1) self._client.register(event, cbk_succeed, uid=1, replace=True) self._client.emit(event, None) @@ -106,9 +117,15 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d1 = defer.Deferred() - cbk1 = lambda event, _: callFromThread(d1.callback, event) + + def cbk1(event, _): + return callFromThread(d1.callback, event) + d2 = defer.Deferred() - cbk2 = lambda event, _: callFromThread(d2.callback, event) + + def cbk2(event, _): + return d2.callback(event) + self._client.register(event, cbk1) self._client.register(event, cbk2) self._client.emit(event, None) @@ -121,8 +138,10 @@ class EventsGenericClientTestCase(object): """ event = catalog.CLIENT_UID d = defer.Deferred() + def cbk(events, _): callFromThread(d.callback, event) + self._client.register(event, cbk) self._client.emit(event, None) return d diff --git a/src/leap/common/zmq_utils.py b/src/leap/common/zmq_utils.py index 19625b9..0a781de 100644 --- a/src/leap/common/zmq_utils.py +++ b/src/leap/common/zmq_utils.py @@ -101,5 +101,3 @@ def maybe_create_and_get_certificates(basedir, name): mkdir_p(public_keys_dir) shutil.move(old_public_key, new_public_key) return zmq.auth.load_certificate(private_key) - - -- cgit v1.2.3 From 301be892a67f88b8b9479531003403a77b36a5f2 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Wed, 22 Jul 2015 19:17:03 -0300 Subject: [tests] set environment for certs test --- src/leap/common/tests/test_certs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/tests/test_certs.py b/src/leap/common/tests/test_certs.py index 209e051..8ebc0f4 100644 --- a/src/leap/common/tests/test_certs.py +++ b/src/leap/common/tests/test_certs.py @@ -43,10 +43,10 @@ CERT_NOT_AFTER = (2023, 9, 1, 17, 52, 16, 4, 244, 0) class CertsTest(BaseLeapTest): def setUp(self): - pass + self.setUpEnv() def tearDown(self): - pass + self.tearDownEnv() def test_should_redownload_if_no_cert(self): self.assertTrue(certs.should_redownload(certfile="")) -- cgit v1.2.3 From 5502318835626527d9818c360c7fd2b1a4b01145 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Wed, 22 Jul 2015 23:55:39 -0300 Subject: [tests] implemented http feature according to test Two test cases were broken and were implemented here: The first was that HTTPClient should share the connection between clients if a pool was not passed explicitly. If you initialize an HTTPClient without a pool, it will reuse a pool created on the class. The second was that you should be able to pass to the HTTPCLient a pool on initialization. Added that possibility and fixed the tests accordingly --- src/leap/common/http.py | 98 +++++++++++++++++++++----------------- src/leap/common/tests/test_http.py | 7 ++- 2 files changed, 59 insertions(+), 46 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index 56938b4..f67507d 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -56,6 +56,50 @@ __all__ = ["HTTPClient"] DEFAULT_HTTP_TIMEOUT = 30 # seconds +class _HTTP11ClientFactory(HTTP11ClientFactory): + """ + A timeout-able HTTP 1.1 client protocol factory. + """ + + def __init__(self, quiescentCallback, timeout): + """ + :param quiescentCallback: The quiescent callback to be passed to + protocol instances, used to return them to + the connection pool. + :type quiescentCallback: callable(Protocol) + :param timeout: The timeout, in seconds, for requests made by + protocols created by this factory. + :type timeout: float + """ + HTTP11ClientFactory.__init__(self, quiescentCallback) + self._timeout = timeout + + def buildProtocol(self, _): + """ + Build the HTTP 1.1 client protocol. + """ + return _HTTP11ClientProtocol(self._quiescentCallback, self._timeout) + + +class _HTTPConnectionPool(HTTPConnectionPool): + """ + A timeout-able HTTP connection pool. + """ + + _factory = _HTTP11ClientFactory + + def __init__(self, reactor, persistent, timeout, maxPersistentPerHost=10): + HTTPConnectionPool.__init__(self, reactor, persistent=persistent) + self.maxPersistentPerHost = maxPersistentPerHost + self._timeout = timeout + + def _newConnection(self, key, endpoint): + def quiescentCallback(protocol): + self._putConnection(key, protocol) + factory = self._factory(quiescentCallback, timeout=self._timeout) + return endpoint.connect(factory) + + class HTTPClient(object): """ HTTP client done the twisted way, with a main focus on pinning the SSL @@ -69,7 +113,14 @@ class HTTPClient(object): in order to avoid resource abuse on huge requests batches. """ - def __init__(self, cert_file=None, timeout=DEFAULT_HTTP_TIMEOUT): + _pool = _HTTPConnectionPool( + reactor, + persistent=True, + timeout = DEFAULT_HTTP_TIMEOUT, + maxPersistentPerHost=10 + ) + + def __init__(self, cert_file=None, timeout=DEFAULT_HTTP_TIMEOUT, pool=None): """ Init the HTTP client @@ -84,7 +135,7 @@ class HTTPClient(object): """ self._timeout = timeout - self._pool = self._createPool() + self._pool = pool if pool is not None else self._pool self._agent = Agent( reactor, get_compatible_ssl_context_factory(cert_file), @@ -278,46 +329,3 @@ class _HTTP11ClientProtocol(HTTP11ClientProtocol): HTTP11ClientProtocol.dataReceived(self, bytes) if self._timeoutCall and self._timeoutCall.active(): self._timeoutCall.reset(self._timeout) - - -class _HTTP11ClientFactory(HTTP11ClientFactory): - """ - A timeout-able HTTP 1.1 client protocol factory. - """ - - def __init__(self, quiescentCallback, timeout): - """ - :param quiescentCallback: The quiescent callback to be passed to - protocol instances, used to return them to - the connection pool. - :type quiescentCallback: callable(Protocol) - :param timeout: The timeout, in seconds, for requests made by - protocols created by this factory. - :type timeout: float - """ - HTTP11ClientFactory.__init__(self, quiescentCallback) - self._timeout = timeout - - def buildProtocol(self, _): - """ - Build the HTTP 1.1 client protocol. - """ - return _HTTP11ClientProtocol(self._quiescentCallback, self._timeout) - - -class _HTTPConnectionPool(HTTPConnectionPool): - """ - A timeout-able HTTP connection pool. - """ - - _factory = _HTTP11ClientFactory - - def __init__(self, reactor, persistent, timeout): - HTTPConnectionPool.__init__(self, reactor, persistent=persistent) - self._timeout = timeout - - def _newConnection(self, key, endpoint): - def quiescentCallback(protocol): - self._putConnection(key, protocol) - factory = self._factory(quiescentCallback, timeout=self._timeout) - return endpoint.connect(factory) diff --git a/src/leap/common/tests/test_http.py b/src/leap/common/tests/test_http.py index e240ca3..a586fd1 100644 --- a/src/leap/common/tests/test_http.py +++ b/src/leap/common/tests/test_http.py @@ -47,7 +47,12 @@ class HTTPClientTest(BaseLeapTest): self.assertEquals(client._agent._pool, client2._agent._pool, "Pool was not reused by default") def test_agent_can_have_dedicated_custom_pool(self): - custom_pool = http.createPool(maxPersistentPerHost=42, persistent=False) + custom_pool = http._HTTPConnectionPool( + None, + timeout=10, + maxPersistentPerHost=42, + persistent=False + ) self.assertEquals(custom_pool.maxPersistentPerHost, 42, "Custom persistent connections limit is not being respected") self.assertFalse(custom_pool.persistent, -- cgit v1.2.3 From 486da2654c63262f0dbc2d603125f0c8c5c4ea74 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Thu, 23 Jul 2015 00:00:39 -0300 Subject: [tests] fixed events tests The events tests check for register and emit signals, but because the flag set_events_enabled was False by default in the tests, no signals were being emitted. I added the flag to the setUp and tearDown of the tests, they are still very slow but at least they are passing now --- src/leap/common/tests/test_events.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/leap/common') diff --git a/src/leap/common/tests/test_events.py b/src/leap/common/tests/test_events.py index e2a918f..8f08eeb 100644 --- a/src/leap/common/tests/test_events.py +++ b/src/leap/common/tests/test_events.py @@ -26,6 +26,7 @@ from twisted.internet import defer from leap.common.events import server from leap.common.events import client +from leap.common.events import flags from leap.common.events import txclient from leap.common.events import catalog from leap.common.events.errors import CallbackAlreadyRegisteredError @@ -44,10 +45,12 @@ class EventsGenericClientTestCase(object): self._client.configure_client( emit_addr="tcp://127.0.0.1:%d" % self._server.pull_port, reg_addr="tcp://127.0.0.1:%d" % self._server.pub_port) + flags.set_events_enabled(True) def tearDown(self): self._client.shutdown() self._server.shutdown() + flags.set_events_enabled(False) # wait a bit for sockets to close properly time.sleep(0.1) -- cgit v1.2.3 From e8d54bd3dc4b5f0327a30c0a6848dd832beb7da0 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Thu, 23 Jul 2015 00:09:03 -0300 Subject: [style] fixed pep8 warnings on http and test events --- src/leap/common/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index f67507d..8e8d3d9 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -116,7 +116,7 @@ class HTTPClient(object): _pool = _HTTPConnectionPool( reactor, persistent=True, - timeout = DEFAULT_HTTP_TIMEOUT, + timeout=DEFAULT_HTTP_TIMEOUT, maxPersistentPerHost=10 ) -- cgit v1.2.3 From 16242a1ebbac7ee0b38d3199b2a1317297d4506b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jul 2015 16:36:12 -0400 Subject: [tests] fix initialization of basetest case --- src/leap/common/testing/test_basetest.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/testing/test_basetest.py b/src/leap/common/testing/test_basetest.py index cf0962d..ec42a62 100644 --- a/src/leap/common/testing/test_basetest.py +++ b/src/leap/common/testing/test_basetest.py @@ -83,12 +83,10 @@ class TestInitBaseLeapTest(BaseLeapTest): """ def setUp(self): - """nuke it""" - pass + self.setUpEnv() def tearDown(self): - """nuke it""" - pass + self.tearDownEnv() def test_path_is_changed(self): """tests whether we have changed the PATH env var""" -- cgit v1.2.3 From a119dd4fa2fc4a14577fd2d6e32dff950d934193 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 28 Jul 2015 11:41:32 -0400 Subject: [style] more pep8 cleanup --- src/leap/common/certs.py | 3 ++- src/leap/common/http.py | 3 ++- src/leap/common/tests/test_http.py | 18 +++++++++++++----- 3 files changed, 17 insertions(+), 7 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/certs.py b/src/leap/common/certs.py index c8e0743..37ede8e 100644 --- a/src/leap/common/certs.py +++ b/src/leap/common/certs.py @@ -192,6 +192,7 @@ def get_compatible_ssl_context_factory(cert_path=None): return policy else: raise Exception((""" - Twisted 14.0.2 is needed in order to have secure Client Web SSL Contexts, not %s + Twisted 14.0.2 is needed in order to have secure + Client Web SSL Contexts, not %s See: http://twistedmatrix.com/trac/ticket/7647 """) % (twisted.version.base())) diff --git a/src/leap/common/http.py b/src/leap/common/http.py index 8e8d3d9..1e7ded7 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -120,7 +120,8 @@ class HTTPClient(object): maxPersistentPerHost=10 ) - def __init__(self, cert_file=None, timeout=DEFAULT_HTTP_TIMEOUT, pool=None): + def __init__(self, cert_file=None, + timeout=DEFAULT_HTTP_TIMEOUT, pool=None): """ Init the HTTP client diff --git a/src/leap/common/tests/test_http.py b/src/leap/common/tests/test_http.py index a586fd1..f44550f 100644 --- a/src/leap/common/tests/test_http.py +++ b/src/leap/common/tests/test_http.py @@ -43,8 +43,11 @@ class HTTPClientTest(BaseLeapTest): def test_agents_sharing_pool_by_default(self): client = http.HTTPClient() client2 = http.HTTPClient(TEST_CERT_PEM) - self.assertNotEquals(client._agent, client2._agent, "Expected dedicated agents") - self.assertEquals(client._agent._pool, client2._agent._pool, "Pool was not reused by default") + self.assertNotEquals( + client._agent, client2._agent, "Expected dedicated agents") + self.assertEquals( + client._agent._pool, client2._agent._pool, + "Pool was not reused by default") def test_agent_can_have_dedicated_custom_pool(self): custom_pool = http._HTTPConnectionPool( @@ -54,14 +57,19 @@ class HTTPClientTest(BaseLeapTest): persistent=False ) self.assertEquals(custom_pool.maxPersistentPerHost, 42, - "Custom persistent connections limit is not being respected") + "Custom persistent connections " + "limit is not being respected") self.assertFalse(custom_pool.persistent, "Custom persistence is not being respected") default_client = http.HTTPClient() custom_client = http.HTTPClient(pool=custom_pool) - self.assertNotEquals(default_client._agent, custom_client._agent, "No agent reuse is expected") - self.assertEquals(custom_pool, custom_client._agent._pool, "Custom pool usage was not respected") + self.assertNotEquals( + default_client._agent, custom_client._agent, + "No agent reuse is expected") + self.assertEquals( + custom_pool, custom_client._agent._pool, + "Custom pool usage was not respected") if __name__ == "__main__": unittest.main() -- cgit v1.2.3 From 08779379337ff77729c9d2ebdb6f4aaf486d38a7 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 28 Jul 2015 16:26:14 -0300 Subject: [bug] Consider events flag when ensuring client Change EventsClientThread behavior so it won't start anymore if the events flag is set to False --- src/leap/common/events/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index 1744341..8d8d522 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -466,7 +466,7 @@ class EventsClientThread(threading.Thread, EventsClient): Make sure the events client thread is started. """ with self._lock: - if not self.is_alive(): + if flags.EVENTS_ENABLED and not self.is_alive(): self.daemon = True self.start() self._initialized.wait() -- cgit v1.2.3 From e47f4bcc0eb9c3d08c649adcb62b0325f439113e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 28 Jul 2015 16:48:04 -0300 Subject: [test] set flag before starting client Test client will only start with flag set to True. Change EventsGenericClientTestCase to set the flag on the first line of setUp. --- src/leap/common/tests/test_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/common') diff --git a/src/leap/common/tests/test_events.py b/src/leap/common/tests/test_events.py index 8f08eeb..2ad097e 100644 --- a/src/leap/common/tests/test_events.py +++ b/src/leap/common/tests/test_events.py @@ -39,13 +39,13 @@ if 'DEBUG' in os.environ: class EventsGenericClientTestCase(object): def setUp(self): + flags.set_events_enabled(True) self._server = server.ensure_server( emit_addr="tcp://127.0.0.1:0", reg_addr="tcp://127.0.0.1:0") self._client.configure_client( emit_addr="tcp://127.0.0.1:%d" % self._server.pull_port, reg_addr="tcp://127.0.0.1:%d" % self._server.pub_port) - flags.set_events_enabled(True) def tearDown(self): self._client.shutdown() -- cgit v1.2.3 From 2e911bd0c949b7f42824ed87c467b1ac0919224a Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 29 Jul 2015 13:30:41 -0300 Subject: [refactor] Extract flags check to caller Checking was done inside of emit method. Doing on emit function at a module level makes it cleaner with less lines inside of check. --- src/leap/common/events/client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index 8d8d522..4790fc3 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -170,10 +170,9 @@ class EventsClient(object): :param content: The content of the event. :type content: list """ - if flags.EVENTS_ENABLED: - logger.debug("Emitting event: (%s, %s)" % (event, content)) - payload = str(event) + b'\0' + pickle.dumps(content) - self._send(payload) + logger.debug("Emitting event: (%s, %s)" % (event, content)) + payload = str(event) + b'\0' + pickle.dumps(content) + self._send(payload) def _handle_event(self, event, content): """ @@ -537,7 +536,8 @@ def emit(event, *content): :param content: The content of the event. :type content: list """ - return EventsClientThread.instance().emit(event, *content) + if flags.EVENTS_ENABLED: + return EventsClientThread.instance().emit(event, *content) def instance(): -- cgit v1.2.3 From 2d9ce114daeaf6c4c193079b018576ad1f4f8f28 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 29 Jul 2015 13:34:08 -0300 Subject: [bug] register and unregister controlled by flag Since register and unregister cant be used without full zmq initialization, it should make sense to also check flag for them. --- src/leap/common/events/client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index 4790fc3..e085f5b 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -508,8 +508,9 @@ def register(event, callback, uid=None, replace=False): :raises CallbackAlreadyRegisteredError: when there's already a callback identified by the given uid and replace is False. """ - return EventsClientThread.instance().register( - event, callback, uid=uid, replace=replace) + if flags.EVENTS_ENABLED: + return EventsClientThread.instance().register( + event, callback, uid=uid, replace=replace) def unregister(event, uid=None): @@ -524,7 +525,8 @@ def unregister(event, uid=None): :param uid: The callback uid. :type uid: str """ - return EventsClientThread.instance().unregister(event, uid=uid) + if flags.EVENTS_ENABLED: + return EventsClientThread.instance().unregister(event, uid=uid) def emit(event, *content): -- cgit v1.2.3 From 0a8f455965fff778d993912f1dc11a77db264a93 Mon Sep 17 00:00:00 2001 From: Bruno Wagner Date: Tue, 4 Aug 2015 18:44:21 -0300 Subject: [bug] HTTP timeout was not being cleared on abort In case the http client loses connection, it has to clear it's timeout or the reactor will be left in a dirty state Fixing this solves a problem with some of the tests in Soledad that were trying to run on a dirty reactor --- src/leap/common/http.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index 1e7ded7..cda8ee8 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -302,23 +302,16 @@ class _HTTP11ClientProtocol(HTTP11ClientProtocol): """ Cancel the request timeout, when it's finished. """ - if self._timeoutCall.active(): + if self._timeoutCall and self._timeoutCall.active(): self._timeoutCall.cancel() self._timeoutCall = None - def _finishResponse_WAITING(self, rest): - """ - Cancel the timeout when finished receiving the response. - """ - self._cancelTimeout() - HTTP11ClientProtocol._finishResponse_WAITING(self, rest) - - def _finishResponse_TRANSMITTING(self, rest): + def _finishResponse(self, rest): """ Cancel the timeout when finished receiving the response. """ self._cancelTimeout() - HTTP11ClientProtocol._finishResponse_TRANSMITTING(self, rest) + HTTP11ClientProtocol._finishResponse(self, rest) def dataReceived(self, bytes): """ @@ -330,3 +323,7 @@ class _HTTP11ClientProtocol(HTTP11ClientProtocol): HTTP11ClientProtocol.dataReceived(self, bytes) if self._timeoutCall and self._timeoutCall.active(): self._timeoutCall.reset(self._timeout) + + def connectionLost(self, reason): + self._cancelTimeout() + return HTTP11ClientProtocol.connectionLost(self, reason) -- cgit v1.2.3 From 8965be42773e3e6c7e154f2c1a27f4e34031ae91 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 11 Aug 2015 11:44:42 -0300 Subject: [feature] allow passing callback to http client --- src/leap/common/http.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index cda8ee8..6fc10b4 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -31,6 +31,7 @@ except ImportError: from leap.common.certs import get_compatible_ssl_context_factory +from leap.common.check import leap_assert from zope.interface import implements @@ -150,7 +151,7 @@ class HTTPClient(object): pool.maxPersistentPerHost = maxPersistentPerHost return pool - def _request(self, url, method, body, headers): + def _request(self, url, method, body, headers, callback): """ Perform an HTTP request. @@ -162,6 +163,9 @@ class HTTPClient(object): :type body: str :param headers: The headers of the request. :type headers: dict + :param callback: A callback to be added to the request's deferred + callback chain. + :type callback: callable :return: A deferred that fires with the body of the request. :rtype: twisted.internet.defer.Deferred @@ -170,14 +174,21 @@ class HTTPClient(object): body = _StringBodyProducer(body) d = self._agent.request( method, url, headers=Headers(headers), bodyProducer=body) - d.addCallback(readBody) + d.addCallback(callback) return d - def request(self, url, method='GET', body=None, headers={}): + def request(self, url, method='GET', body=None, headers={}, + callback=readBody): """ Perform an HTTP request, but limit the maximum amount of concurrent connections. + May be passed a callback to be added to the request's deferred + callback chain. The callback is expected to receive the response of + the request and may do whatever it wants with the response. By + default, if no callback is passed, we will use a simple body reader + which returns a deferred that is fired with the body of the response. + :param url: The URL for the request. :type url: str :param method: The HTTP method of the request. @@ -186,11 +197,18 @@ class HTTPClient(object): :type body: str :param headers: The headers of the request. :type headers: dict + :param callback: A callback to be added to the request's deferred + callback chain. + :type callback: callable :return: A deferred that fires with the body of the request. :rtype: twisted.internet.defer.Deferred """ - return self._semaphore.run(self._request, url, method, body, headers) + leap_assert( + callable(callback), + message="The callback parameter should be a callable!") + return self._semaphore.run(self._request, url, method, body, headers, + callback) def close(self): """ -- cgit v1.2.3 From 97096fa2c22e96ec238c38ad3befdd052b5028cf Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 17 Aug 2015 19:15:05 -0400 Subject: [style] pep8 fix --- src/leap/common/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/http.py b/src/leap/common/http.py index 6fc10b4..0dee3a2 100644 --- a/src/leap/common/http.py +++ b/src/leap/common/http.py @@ -20,6 +20,7 @@ Twisted HTTP/HTTPS client. try: import twisted + assert twisted except ImportError: print "*******" print "Twisted is needed to use leap.common.http module" @@ -178,7 +179,7 @@ class HTTPClient(object): return d def request(self, url, method='GET', body=None, headers={}, - callback=readBody): + callback=readBody): """ Perform an HTTP request, but limit the maximum amount of concurrent connections. @@ -208,7 +209,7 @@ class HTTPClient(object): callable(callback), message="The callback parameter should be a callable!") return self._semaphore.run(self._request, url, method, body, headers, - callback) + callback) def close(self): """ -- cgit v1.2.3 From 0bcd8ae08d9b0d42e8cc57e34d301c192a582d60 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 28 Aug 2015 11:00:26 -0400 Subject: freeze debian version --- src/leap/common/_version.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index 3500492..c46fd83 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -1,3 +1,13 @@ -version_version = '0.3.9' -version_full = '3de1eeba83d50793283d65ba4566dd1611ee9d4b' +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.7+) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '0.4.2' +version_full = '8fa97c02b5f07f896e52d9bb272128f267af04ea' + + +def get_versions(default={}, verbose=False): + return {'version': version_version, 'full': version_full} -- cgit v1.2.3 From 9d0900de85d07ec5b7926708c51136810b337158 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 14 Sep 2015 23:06:29 -0400 Subject: [feat] expose async events in api to make simpler the import of the tx client - Related: #7274 --- src/leap/common/events/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/leap/common') diff --git a/src/leap/common/events/__init__.py b/src/leap/common/events/__init__.py index 87ed8ae..18d0ac8 100644 --- a/src/leap/common/events/__init__.py +++ b/src/leap/common/events/__init__.py @@ -39,6 +39,7 @@ import logging import argparse from leap.common.events import client +from leap.common.events import txclient from leap.common.events import server from leap.common.events.flags import set_events_enabled @@ -80,6 +81,10 @@ def register(event, callback, uid=None, replace=False): return client.register(event, callback, uid, replace) +def register_async(event, callback, uid=None, replace=False): + return txclient.register(event, callback, uid, replace) + + def unregister(event, uid=None): """ Unregister callbacks for an event. @@ -95,6 +100,10 @@ def unregister(event, uid=None): return client.unregister(event, uid) +def unregister_async(event, uid=None): + return txclient.unregister(event, uid) + + def emit(event, *content): """ Send an event. @@ -107,6 +116,10 @@ def emit(event, *content): return client.emit(event, *content) +def emit_async(event, *content): + return txclient.emit(event, *content) + + if __name__ == "__main__": def _echo(event, *content): -- cgit v1.2.3 From d4e4d3dc8ff88d98477e1fcd1104d7cd8b9e6612 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Wed, 16 Sep 2015 11:21:01 +0200 Subject: [tests] Make txclient honor flags.EVENTS_ENABLED - Breaks tests in dependent repos like soledad otherwise, because server.key could not be found --- src/leap/common/events/txclient.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/txclient.py b/src/leap/common/events/txclient.py index dfd0533..be4b3ef 100644 --- a/src/leap/common/events/txclient.py +++ b/src/leap/common/events/txclient.py @@ -35,7 +35,7 @@ from leap.common.events.client import EventsClient from leap.common.events.client import configure_client from leap.common.events.server import EMIT_ADDR from leap.common.events.server import REG_ADDR -from leap.common.events import catalog +from leap.common.events import catalog, flags logger = logging.getLogger(__name__) @@ -146,8 +146,9 @@ def register(event, callback, uid=None, replace=False): :raises CallbackAlreadyRegisteredError: when there's already a callback identified by the given uid and replace is False. """ - return EventsTxClient.instance().register( - event, callback, uid=uid, replace=replace) + if flags.EVENTS_ENABLED: + return EventsTxClient.instance().register( + event, callback, uid=uid, replace=replace) def unregister(event, uid=None): @@ -162,7 +163,8 @@ def unregister(event, uid=None): :param uid: The callback uid. :type uid: str """ - return EventsTxClient.instance().unregister(event, uid=uid) + if flags.EVENTS_ENABLED: + return EventsTxClient.instance().unregister(event, uid=uid) def emit(event, *content): @@ -174,7 +176,8 @@ def emit(event, *content): :param content: The content of the event. :type content: list """ - return EventsTxClient.instance().emit(event, *content) + if flags.EVENTS_ENABLED: + return EventsTxClient.instance().emit(event, *content) def shutdown(): -- cgit v1.2.3 From 967763e8d0cc5f70f0760994d87500e30ad724a7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 16 Sep 2015 12:20:22 -0400 Subject: [refactor] move checking flag to main module --- src/leap/common/events/__init__.py | 19 +++++++++++++------ src/leap/common/events/client.py | 13 +++++-------- src/leap/common/events/txclient.py | 11 ++++------- 3 files changed, 22 insertions(+), 21 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/events/__init__.py b/src/leap/common/events/__init__.py index 18d0ac8..f9ad5fa 100644 --- a/src/leap/common/events/__init__.py +++ b/src/leap/common/events/__init__.py @@ -41,6 +41,7 @@ import argparse from leap.common.events import client from leap.common.events import txclient from leap.common.events import server +from leap.common.events import flags from leap.common.events.flags import set_events_enabled from leap.common.events import catalog @@ -78,11 +79,13 @@ def register(event, callback, uid=None, replace=False): :raises CallbackAlreadyRegistered: when there's already a callback identified by the given uid and replace is False. """ - return client.register(event, callback, uid, replace) + if flags.EVENTS_ENABLED: + return client.register(event, callback, uid, replace) def register_async(event, callback, uid=None, replace=False): - return txclient.register(event, callback, uid, replace) + if flags.EVENTS_ENABLED: + return txclient.register(event, callback, uid, replace) def unregister(event, uid=None): @@ -97,11 +100,13 @@ def unregister(event, uid=None): :param uid: The callback uid. :type uid: str """ - return client.unregister(event, uid) + if flags.EVENTS_ENABLED: + return client.unregister(event, uid) def unregister_async(event, uid=None): - return txclient.unregister(event, uid) + if flags.EVENTS_ENABLED: + return txclient.unregister(event, uid) def emit(event, *content): @@ -113,11 +118,13 @@ def emit(event, *content): :param content: The content of the event. :type content: list """ - return client.emit(event, *content) + if flags.EVENTS_ENABLED: + return client.emit(event, *content) def emit_async(event, *content): - return txclient.emit(event, *content) + if flags.EVENTS_ENABLED: + return txclient.emit(event, *content) if __name__ == "__main__": diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index e085f5b..e38e9d3 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -465,7 +465,7 @@ class EventsClientThread(threading.Thread, EventsClient): Make sure the events client thread is started. """ with self._lock: - if flags.EVENTS_ENABLED and not self.is_alive(): + if not self.is_alive(): self.daemon = True self.start() self._initialized.wait() @@ -508,9 +508,8 @@ def register(event, callback, uid=None, replace=False): :raises CallbackAlreadyRegisteredError: when there's already a callback identified by the given uid and replace is False. """ - if flags.EVENTS_ENABLED: - return EventsClientThread.instance().register( - event, callback, uid=uid, replace=replace) + return EventsClientThread.instance().register( + event, callback, uid=uid, replace=replace) def unregister(event, uid=None): @@ -525,8 +524,7 @@ def unregister(event, uid=None): :param uid: The callback uid. :type uid: str """ - if flags.EVENTS_ENABLED: - return EventsClientThread.instance().unregister(event, uid=uid) + return EventsClientThread.instance().unregister(event, uid=uid) def emit(event, *content): @@ -538,8 +536,7 @@ def emit(event, *content): :param content: The content of the event. :type content: list """ - if flags.EVENTS_ENABLED: - return EventsClientThread.instance().emit(event, *content) + return EventsClientThread.instance().emit(event, *content) def instance(): diff --git a/src/leap/common/events/txclient.py b/src/leap/common/events/txclient.py index be4b3ef..f3c183e 100644 --- a/src/leap/common/events/txclient.py +++ b/src/leap/common/events/txclient.py @@ -146,9 +146,8 @@ def register(event, callback, uid=None, replace=False): :raises CallbackAlreadyRegisteredError: when there's already a callback identified by the given uid and replace is False. """ - if flags.EVENTS_ENABLED: - return EventsTxClient.instance().register( - event, callback, uid=uid, replace=replace) + return EventsTxClient.instance().register( + event, callback, uid=uid, replace=replace) def unregister(event, uid=None): @@ -163,8 +162,7 @@ def unregister(event, uid=None): :param uid: The callback uid. :type uid: str """ - if flags.EVENTS_ENABLED: - return EventsTxClient.instance().unregister(event, uid=uid) + return EventsTxClient.instance().unregister(event, uid=uid) def emit(event, *content): @@ -176,8 +174,7 @@ def emit(event, *content): :param content: The content of the event. :type content: list """ - if flags.EVENTS_ENABLED: - return EventsTxClient.instance().emit(event, *content) + return EventsTxClient.instance().emit(event, *content) def shutdown(): -- cgit v1.2.3 From 5b62dbbc0de18342c7b9ab0236e888b01be0ff41 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 24 Sep 2015 15:19:25 -0400 Subject: freeze debian version --- src/leap/common/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index c46fd83..5a1d563 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,8 +5,8 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.4.2' -version_full = '8fa97c02b5f07f896e52d9bb272128f267af04ea' +version_version = '0.4.3' +version_full = 'c84a7358a922b090a25789e76fcbc9a37b344117' def get_versions(default={}, verbose=False): -- cgit v1.2.3 From 459024de9e36aea0813aa01a570b68db7e9c1a26 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Tue, 6 Oct 2015 14:41:18 -0300 Subject: [bug] consider STANDALONE for the paths Since we need to write a file we have to consider whether we are running in 'standalone' mode or not to use the right path prefix. - Related: #7512 --- src/leap/common/config/flags.py | 28 ++++++++++++++++++++++++++++ src/leap/common/events/client.py | 5 ++--- src/leap/common/events/txclient.py | 2 +- src/leap/common/events/zmq_components.py | 4 ++-- 4 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 src/leap/common/config/flags.py (limited to 'src/leap/common') diff --git a/src/leap/common/config/flags.py b/src/leap/common/config/flags.py new file mode 100644 index 0000000..6fd43f6 --- /dev/null +++ b/src/leap/common/config/flags.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# flags.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +This file is meant to be used to store global flags that affect the +application. + +WARNING: You should NOT use this kind of flags unless you're sure of what + you're doing, and someone else tells you that you're right. + Most of the times there is a better and safer alternative. +""" + +# The STANDALONE flag is used to: +# - use a relative or system wide path to find the configuration files. +STANDALONE = False diff --git a/src/leap/common/events/client.py b/src/leap/common/events/client.py index e38e9d3..60d24bc 100644 --- a/src/leap/common/events/client.py +++ b/src/leap/common/events/client.py @@ -47,7 +47,7 @@ try: except ImportError: pass -from leap.common.config import get_path_prefix +from leap.common.config import flags, get_path_prefix from leap.common.zmq_utils import zmq_has_curve from leap.common.zmq_utils import maybe_create_and_get_certificates from leap.common.zmq_utils import PUBLIC_KEYS_PREFIX @@ -55,7 +55,6 @@ from leap.common.zmq_utils import PUBLIC_KEYS_PREFIX from leap.common.events.errors import CallbackAlreadyRegisteredError from leap.common.events.server import EMIT_ADDR from leap.common.events.server import REG_ADDR -from leap.common.events import flags from leap.common.events import catalog @@ -280,7 +279,7 @@ class EventsClientThread(threading.Thread, EventsClient): self._lock = threading.Lock() self._initialized = threading.Event() self._config_prefix = os.path.join( - get_path_prefix(), "leap", "events") + get_path_prefix(flags.STANDALONE), "leap", "events") self._loop = None self._context = None self._push = None diff --git a/src/leap/common/events/txclient.py b/src/leap/common/events/txclient.py index f3c183e..dfd0533 100644 --- a/src/leap/common/events/txclient.py +++ b/src/leap/common/events/txclient.py @@ -35,7 +35,7 @@ from leap.common.events.client import EventsClient from leap.common.events.client import configure_client from leap.common.events.server import EMIT_ADDR from leap.common.events.server import REG_ADDR -from leap.common.events import catalog, flags +from leap.common.events import catalog logger = logging.getLogger(__name__) diff --git a/src/leap/common/events/zmq_components.py b/src/leap/common/events/zmq_components.py index f99c754..729ca90 100644 --- a/src/leap/common/events/zmq_components.py +++ b/src/leap/common/events/zmq_components.py @@ -36,7 +36,7 @@ try: except ImportError: pass -from leap.common.config import get_path_prefix +from leap.common.config import flags, get_path_prefix from leap.common.zmq_utils import zmq_has_curve from leap.common.zmq_utils import maybe_create_and_get_certificates from leap.common.zmq_utils import PUBLIC_KEYS_PREFIX @@ -64,7 +64,7 @@ class TxZmqComponent(object): self._factory = txzmq.ZmqFactory() self._factory.registerForShutdown() if path_prefix is None: - path_prefix = get_path_prefix() + path_prefix = get_path_prefix(flags.STANDALONE) self._config_prefix = os.path.join(path_prefix, "leap", "events") self._connections = [] -- cgit v1.2.3 From c1781d87a76f941c588709405cd5b2634dc6b1e8 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 9 Oct 2015 15:01:13 -0400 Subject: [bug] fix wrong ca_cert path inside bundle -Resolves: #7524 --- src/leap/common/ca_bundle.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/ca_bundle.py b/src/leap/common/ca_bundle.py index 2c41d18..c0ce35f 100644 --- a/src/leap/common/ca_bundle.py +++ b/src/leap/common/ca_bundle.py @@ -21,8 +21,9 @@ If you are packaging Requests, e.g., for a Linux distribution or a managed environment, you can change the definition of where() to return a separately packaged CA bundle. """ -import platform import os.path +import platform +import sys _system = platform.system() @@ -34,12 +35,11 @@ def where(): Return the preferred certificate bundle. :rtype: str """ - # vendored bundle inside Requests, plus some additions of ours - if IS_MAC: - return os.path.join("/Applications", "Bitmask.app", - "Contents", "Resources", - "cacert.pem") - return os.path.join(os.path.dirname(__file__), 'cacert.pem') + if getattr(sys, 'frozen', False): + # we are running in a |PyInstaller| bundle + path = sys._MEIPASS + return os.path.join(path, 'cacert.pem') + return os.path.join(os.path, dirname(__file__), 'cacert.pem') if __name__ == '__main__': print(where()) -- cgit v1.2.3 From c105a8d3b016cc907a908de1a31b445b32b42ca9 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Tue, 13 Oct 2015 13:09:45 -0300 Subject: [bug] fix typo on dirname usage --- src/leap/common/ca_bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/common') diff --git a/src/leap/common/ca_bundle.py b/src/leap/common/ca_bundle.py index c0ce35f..e2a624d 100644 --- a/src/leap/common/ca_bundle.py +++ b/src/leap/common/ca_bundle.py @@ -39,7 +39,7 @@ def where(): # we are running in a |PyInstaller| bundle path = sys._MEIPASS return os.path.join(path, 'cacert.pem') - return os.path.join(os.path, dirname(__file__), 'cacert.pem') + return os.path.join(os.path.dirname(__file__), 'cacert.pem') if __name__ == '__main__': print(where()) -- cgit v1.2.3 From 5af395a5662c63a09cc6db2bacbb495090488d78 Mon Sep 17 00:00:00 2001 From: Folker Bernitt Date: Wed, 21 Oct 2015 10:07:06 +0200 Subject: Workaround for deadlock problem in zmq auth - See https://leap.se/code/issues/7536 - Actual root cause not identified yet --- src/leap/common/events/zmq_components.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/leap/common') diff --git a/src/leap/common/events/zmq_components.py b/src/leap/common/events/zmq_components.py index 729ca90..51de02c 100644 --- a/src/leap/common/events/zmq_components.py +++ b/src/leap/common/events/zmq_components.py @@ -25,6 +25,7 @@ import os import logging import txzmq import re +import time from abc import ABCMeta @@ -154,6 +155,11 @@ class TxZmqComponent(object): :type socket: zmq.Socket """ authenticator = ThreadAuthenticator(self._factory.context) + + # Temporary fix until we understand what the problem is + # See https://leap.se/code/issues/7536 + time.sleep(0.5) + authenticator.start() # XXX do not hardcode this here. authenticator.allow('127.0.0.1') -- cgit v1.2.3 From ef839547f9404dd8ba70bdaa2fec82dfd07cb383 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 29 Oct 2015 12:00:48 -0400 Subject: freeze debian version --- src/leap/common/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/common') diff --git a/src/leap/common/_version.py b/src/leap/common/_version.py index 5a1d563..f5738ea 100644 --- a/src/leap/common/_version.py +++ b/src/leap/common/_version.py @@ -5,8 +5,8 @@ # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '0.4.3' -version_full = 'c84a7358a922b090a25789e76fcbc9a37b344117' +version_version = '0.4.4' +version_full = 'ee0e9cadccd00cb62032d8fc4b322bb6fe3dc7ed' def get_versions(default={}, verbose=False): -- cgit v1.2.3