fetch only if not changed-since config file timestamp

Changing this now to be able to test different providers by
just updating our local config file.

3 files changed, 75 insertions, 18 deletions

diff --git a/pkg/requirements.pip b/pkg/requirements.pip
index e5338744..5664aa5e 100644
--- a/pkg/requirements.pip
+++ b/pkg/requirements.pip
@@ -11,3 +11,4 @@ jsonschema
 srp
 pycrypto
 keyring
+python-dateutil
diff --git a/src/leap/base/config.py b/src/leap/base/config.py
index 0255fbab..321fbdcd 100644
--- a/src/leap/base/config.py
+++ b/src/leap/base/config.py
@@ -5,11 +5,12 @@ import grp
 import json
 import logging
 import socket
-import tempfile
+import time
 import os
 
 logger = logging.getLogger(name=__name__)
 
+from dateutil import parser as dateparser
 import requests
 
 from leap.base import exceptions
@@ -126,16 +127,23 @@ class JSONLeapConfig(BaseLeapConfig):
     # mandatory baseconfig interface
 
     def save(self, to=None):
-        if to is None:
-            to = self.filename
-        folder, filename = os.path.split(to)
-        if folder and not os.path.isdir(folder):
-            mkdir_p(folder)
-        self._config.serialize(to)
-
-    def load(self, fromfile=None, from_uri=None, fetcher=None, verify=False):
+        if self._config.is_dirty():
+            if to is None:
+                to = self.filename
+            folder, filename = os.path.split(to)
+            if folder and not os.path.isdir(folder):
+                mkdir_p(folder)
+            self._config.serialize(to)
+
+    def load(self, fromfile=None, from_uri=None, fetcher=None,
+             force_download=False, verify=False):
+
         if from_uri is not None:
-            fetched = self.fetch(from_uri, fetcher=fetcher, verify=verify)
+            fetched = self.fetch(
+                from_uri,
+                fetcher=fetcher,
+                verify=verify,
+                force_dl=force_download)
             if fetched:
                 return
         if fromfile is None:
@@ -146,33 +154,64 @@ class JSONLeapConfig(BaseLeapConfig):
             logger.error('tried to load config from non-existent path')
             logger.error('Not Found: %s', fromfile)
 
-    def fetch(self, uri, fetcher=None, verify=True):
+    def fetch(self, uri, fetcher=None, verify=True, force_dl=False):
         if not fetcher:
             fetcher = self.fetcher
+
         logger.debug('verify: %s', verify)
         logger.debug('uri: %s', uri)
-        request = fetcher.get(uri, verify=verify)
-        # XXX should send a if-modified-since header
 
-        # XXX get 404, ...
-        # and raise a UnableToFetch...
+        rargs = (uri, )
+        rkwargs = {'verify': verify}
+        headers = {}
+
+        curmtime = self.get_mtime() if not force_dl else None
+        if curmtime:
+            logger.debug('requesting with if-modified-since %s' % curmtime)
+            headers['if-modified-since'] = curmtime
+            rkwargs['headers'] = headers
+
+        #request = fetcher.get(uri, verify=verify)
+        request = fetcher.get(*rargs, **rkwargs)
         request.raise_for_status()
-        fd, fname = tempfile.mkstemp(suffix=".json")
 
-        if request.json:
-            self._config.load(json.dumps(request.json))
+        if request.status_code == 304:
+            logger.debug('...304 Not Changed')
+            # On this point, we have to assume that
+            # we HAD the filename. If that filename is corruct,
+            # we should enforce a force_download in the load
+            # method above.
+            self._config.load(fromfile=self.filename)
+            return True
 
+        if request.json:
+            mtime = None
+            last_modified = request.headers.get('last-modified', None)
+            if last_modified:
+                _mtime = dateparser.parse(last_modified)
+                mtime = int(_mtime.strftime("%s"))
+            self._config.load(json.dumps(request.json), mtime=mtime)
+            self._config.set_dirty()
         else:
             # not request.json
             # might be server did not announce content properly,
             # let's try deserializing all the same.
             try:
                 self._config.load(request.content)
+                self._config.set_dirty()
             except ValueError:
                 raise eipexceptions.LeapBadConfigFetchedError
 
         return True
 
+    def get_mtime(self):
+        try:
+            _mtime = os.stat(self.filename)[8]
+            mtime = time.strftime("%c GMT", time.gmtime(_mtime))
+            return mtime
+        except OSError:
+            return None
+
     def get_config(self):
         return self._config.config
 
diff --git a/src/leap/base/pluggableconfig.py b/src/leap/base/pluggableconfig.py
index b8615ad8..34c1e060 100644
--- a/src/leap/base/pluggableconfig.py
+++ b/src/leap/base/pluggableconfig.py
@@ -180,6 +180,8 @@ class PluggableConfig(object):
         self.adaptors = adaptors
         self.types = types
         self._format = format
+        self.mtime = None
+        self.dirty = False
 
     @property
     def option_dict(self):
@@ -319,6 +321,13 @@ class PluggableConfig(object):
         serializable = self.prep_value(config)
         adaptor.write(serializable, filename)
 
+        if self.mtime:
+            self.touch_mtime(filename)
+
+    def touch_mtime(self, filename):
+        mtime = self.mtime
+        os.utime(filename, (mtime, mtime))
+
     def deserialize(self, string=None, fromfile=None, format=None):
         """
         load configuration from a file or string
@@ -364,6 +373,12 @@ class PluggableConfig(object):
             content = _try_deserialize()
         return content
 
+    def set_dirty(self):
+        self.dirty = True
+
+    def is_dirty(self):
+        return self.dirty
+
     def load(self, *args, **kwargs):
         """
         load from string or file
@@ -373,6 +388,8 @@ class PluggableConfig(object):
         """
         string = args[0] if args else None
         fromfile = kwargs.get("fromfile", None)
+        mtime = kwargs.pop("mtime", None)
+        self.mtime = mtime
         content = None
 
         # start with defaults, so we can