From 18be85f13abc6bc94a3725950ec16ad1adec0ab8 Mon Sep 17 00:00:00 2001 From: kali Date: Tue, 11 Dec 2012 01:40:05 +0900 Subject: fetch only if not changed-since config file timestamp Changing this now to be able to test different providers by just updating our local config file. --- src/leap/base/config.py | 75 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 18 deletions(-) (limited to 'src/leap/base/config.py') diff --git a/src/leap/base/config.py b/src/leap/base/config.py index 0255fbab..321fbdcd 100644 --- a/src/leap/base/config.py +++ b/src/leap/base/config.py @@ -5,11 +5,12 @@ import grp import json import logging import socket -import tempfile +import time import os logger = logging.getLogger(name=__name__) +from dateutil import parser as dateparser import requests from leap.base import exceptions @@ -126,16 +127,23 @@ class JSONLeapConfig(BaseLeapConfig): # mandatory baseconfig interface def save(self, to=None): - if to is None: - to = self.filename - folder, filename = os.path.split(to) - if folder and not os.path.isdir(folder): - mkdir_p(folder) - self._config.serialize(to) - - def load(self, fromfile=None, from_uri=None, fetcher=None, verify=False): + if self._config.is_dirty(): + if to is None: + to = self.filename + folder, filename = os.path.split(to) + if folder and not os.path.isdir(folder): + mkdir_p(folder) + self._config.serialize(to) + + def load(self, fromfile=None, from_uri=None, fetcher=None, + force_download=False, verify=False): + if from_uri is not None: - fetched = self.fetch(from_uri, fetcher=fetcher, verify=verify) + fetched = self.fetch( + from_uri, + fetcher=fetcher, + verify=verify, + force_dl=force_download) if fetched: return if fromfile is None: @@ -146,33 +154,64 @@ class JSONLeapConfig(BaseLeapConfig): logger.error('tried to load config from non-existent path') logger.error('Not Found: %s', fromfile) - def fetch(self, uri, fetcher=None, verify=True): + def fetch(self, uri, fetcher=None, verify=True, force_dl=False): if not fetcher: fetcher = self.fetcher + logger.debug('verify: %s', verify) logger.debug('uri: %s', uri) - request = fetcher.get(uri, verify=verify) - # XXX should send a if-modified-since header - # XXX get 404, ... - # and raise a UnableToFetch... + rargs = (uri, ) + rkwargs = {'verify': verify} + headers = {} + + curmtime = self.get_mtime() if not force_dl else None + if curmtime: + logger.debug('requesting with if-modified-since %s' % curmtime) + headers['if-modified-since'] = curmtime + rkwargs['headers'] = headers + + #request = fetcher.get(uri, verify=verify) + request = fetcher.get(*rargs, **rkwargs) request.raise_for_status() - fd, fname = tempfile.mkstemp(suffix=".json") - if request.json: - self._config.load(json.dumps(request.json)) + if request.status_code == 304: + logger.debug('...304 Not Changed') + # On this point, we have to assume that + # we HAD the filename. If that filename is corruct, + # we should enforce a force_download in the load + # method above. + self._config.load(fromfile=self.filename) + return True + if request.json: + mtime = None + last_modified = request.headers.get('last-modified', None) + if last_modified: + _mtime = dateparser.parse(last_modified) + mtime = int(_mtime.strftime("%s")) + self._config.load(json.dumps(request.json), mtime=mtime) + self._config.set_dirty() else: # not request.json # might be server did not announce content properly, # let's try deserializing all the same. try: self._config.load(request.content) + self._config.set_dirty() except ValueError: raise eipexceptions.LeapBadConfigFetchedError return True + def get_mtime(self): + try: + _mtime = os.stat(self.filename)[8] + mtime = time.strftime("%c GMT", time.gmtime(_mtime)) + return mtime + except OSError: + return None + def get_config(self): return self._config.config -- cgit v1.2.3 From 04d423e2a89034dfb86fe305108162fd2a696079 Mon Sep 17 00:00:00 2001 From: kali Date: Wed, 12 Dec 2012 03:29:31 +0900 Subject: tests for openvpn options and make the rest of tests pass after some changes in this branch (dirtyness in config files) --- src/leap/base/config.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) (limited to 'src/leap/base/config.py') diff --git a/src/leap/base/config.py b/src/leap/base/config.py index 321fbdcd..b307ad05 100644 --- a/src/leap/base/config.py +++ b/src/leap/base/config.py @@ -126,14 +126,33 @@ class JSONLeapConfig(BaseLeapConfig): # mandatory baseconfig interface - def save(self, to=None): - if self._config.is_dirty(): + def save(self, to=None, force=False): + """ + force param will skip the dirty check. + :type force: bool + """ + # XXX this force=True does not feel to right + # but still have to look for a better way + # of dealing with dirtiness and the + # trick of loading remote config only + # when newer. + + if force: + do_save = True + else: + do_save = self._config.is_dirty() + + if do_save: if to is None: to = self.filename folder, filename = os.path.split(to) if folder and not os.path.isdir(folder): mkdir_p(folder) self._config.serialize(to) + return True + + else: + return False def load(self, fromfile=None, from_uri=None, fetcher=None, force_download=False, verify=False): -- cgit v1.2.3 From ef3e6f7badf4d477e796f8522b4d79b554f18f93 Mon Sep 17 00:00:00 2001 From: kali Date: Tue, 18 Dec 2012 06:09:57 +0900 Subject: back-compat fix for requests api change --- src/leap/base/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/leap/base/config.py') diff --git a/src/leap/base/config.py b/src/leap/base/config.py index b307ad05..438d1993 100644 --- a/src/leap/base/config.py +++ b/src/leap/base/config.py @@ -209,7 +209,12 @@ class JSONLeapConfig(BaseLeapConfig): if last_modified: _mtime = dateparser.parse(last_modified) mtime = int(_mtime.strftime("%s")) - self._config.load(json.dumps(request.json), mtime=mtime) + if callable(request.json): + _json = request.json() + else: + # back-compat + _json = request.json + self._config.load(json.dumps(_json), mtime=mtime) self._config.set_dirty() else: # not request.json -- cgit v1.2.3