diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/thandy/ClientCLI.py | 63 | ||||
| -rw-r--r-- | lib/thandy/formats.py | 13 | ||||
| -rw-r--r-- | lib/thandy/packagesys/ExePackages.py | 37 | ||||
| -rw-r--r-- | lib/thandy/packagesys/PackageDB.py | 45 | ||||
| -rw-r--r-- | lib/thandy/packagesys/PackageSystem.py | 35 | ||||
| -rw-r--r-- | lib/thandy/packagesys/RPMPackages.py | 39 | ||||
| -rw-r--r-- | lib/thandy/repository.py | 33 |
7 files changed, 222 insertions, 43 deletions
diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 7d29de7..7818c6d 100644 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -4,32 +4,78 @@ import getopt import logging import os import sys +import time +import thandy.formats import thandy.util import thandy.repository import thandy.download import thandy.master_keys +import thandy.packagesys.PackageSystem def update(args): repoRoot = thandy.util.userFilename("cache") - options, args = getopt.getopt(args, "", [ "repo=", "no-download" ]) + options, args = getopt.getopt(args, "", [ "repo=", "no-download", + "loop", "no-packagesys", + "install"]) download = True + keep_looping = False + use_packagesys = True + install = False for o, v in options: if o == '--repo': repoRoot = v elif o == "--no-download": download = False + elif o == '--loop': + keep_looping = True + elif o == '--no-packagesys': + use_packagesys = False + elif o == '--install': + install = True repo = thandy.repository.LocalRepository(repoRoot) + packagesys = None + if use_packagesys: + packagesys = thandy.packagesys.PackageSystem.PackageMetasystem.create(repo) while True: hashes = {} + installable = {} logging.info("Checking for files to update.") - files = repo.getFilesToUpdate(trackingBundles=args, hashDict=hashes) + files = repo.getFilesToUpdate(trackingBundles=args, hashDict=hashes, + pkgSystems=packagesys, + installableDict=installable) + + if installable and not files: + logging.notice("Ready to install files: %s", + ", ".join(sorted(installable.keys()))) + if install: + # XXXX handle ordering + for h in installable.values(): + h.install() + return + + elif not files: + logging.info("No files to download") + if not keep_looping: + return + + ts = repo.getTimestampFile().get() + age = time.time() - thandy.formats.parseTime(ts['at']) + delay = thandy.repository.MAX_TIMESTAMP_AGE - age + if delay > 3600: + delay = 3600 + elif delay < 0: + delay = 300 + logging.info("Will check again in %s seconds", delay) + time.sleep(delay) + continue + logging.info("Files to download are: %s", ", ".join(sorted(files))) - if not download or not files: + if not download: return mirrorlist = repo.getMirrorlistFile().get() @@ -59,18 +105,13 @@ def update(args): logging.info("All downloads finished.") -# Check my repository - -# Tell me what I need to download - -# Download stuff - # Tell me what to install. def usage(): print "Known commands:" - print " update [--repo=repository] [--no-download]" + print " update [--repo=repository] [--no-download] [--loop]" + print " [--no-packagesys] [--install]" sys.exit(1) def main(): @@ -81,7 +122,7 @@ def main(): usage() cmd = sys.argv[1] args = sys.argv[2:] - if cmd in [ "update", "geturls" ]: + if cmd in [ "update" ]: globals()[cmd](args) else: usage() diff --git a/lib/thandy/formats.py b/lib/thandy/formats.py index 7ae6080..4afbd00 100644 --- a/lib/thandy/formats.py +++ b/lib/thandy/formats.py @@ -451,7 +451,8 @@ PACKAGE_SCHEMA = S.Obj( version=VERSION_SCHEMA, format=S.Obj(), ts=TIME_SCHEMA, - files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])), + files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA], + allowMore=True)), shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()), longdesc=S.DictOf(S.AnyStr(), S.AnyStr())) @@ -612,23 +613,29 @@ def makePackageObj(config_fname, package_fname): 'format', 'location', 'relpath', - ], (), preload) + ], ['rpm_version', 'exe_args'], preload) f = open(package_fname, 'rb') digest = getFileDigest(f) # Check fields! + extra = {} result = { '_type' : "Package", 'ts' : formatTime(time.time()), 'name' : r['name'], 'location' : r['location'], #DOCDOC 'version' : r['version'], 'format' : r['format'], - 'files' : [ [ r['relpath'], formatHash(digest) ] ], + 'files' : [ [ r['relpath'], formatHash(digest), extra ] ], 'shortdesc' : shortDescs, 'longdesc' : longDescs } + if format == 'rpm' and r.get('rpm_version'): + extra['rpm_version'] = r['rpm_version'] + elif format == 'exe' and r.get('exe_args') != None: + extra['exe_args'] = r['exe_args'] + PACKAGE_SCHEMA.checkMatch(result) return result diff --git a/lib/thandy/packagesys/ExePackages.py b/lib/thandy/packagesys/ExePackages.py index 1688da3..4989f3a 100644 --- a/lib/thandy/packagesys/ExePackages.py +++ b/lib/thandy/packagesys/ExePackages.py @@ -1,15 +1,38 @@ # Copyright 2008 The Tor Project, Inc. See LICENSE for licensing information. +import subprocess + +import thandy.util import thandy.packagesys.PackageSystem as ps import thandy.packagesys.PackageDB as pdb class ExePackageSystem(pdb.DBBackedPackageSystem): + def __init__(self, repo): + self._repo = repo def getName(self): - return "executable" - - def packageHandleFromJSON(self, json): - raise NotImplemented() #XXXX???? + return "exe" + + def packageHandlesFromJSON(self, pkg): + if pkg['format'] != 'exe': + raise thandy.FormatException() + + handles = [] + for entry in pkg['files']: + if len(entry) < 3: + continue + rp, h, extra = entry[:3] + version = package['version'] + + handles.append( + ExePackageHandle(self.getDB(), + package['name'], + version, + [], # filelist not implemented in this. + rp, + self._repo.getFilename(rp), + extra['exe_args'])) + return handles def canBeAutomatic(self): return True @@ -18,12 +41,16 @@ class ExePackageSystem(pdb.DBBackedPackageSystem): return True class ExePackageHandle(pdb.DBBackedPackageHandle): - def __init__(self, packageDB, name, version, filelist, filename, + def __init__(self, packageDB, name, version, filelist, relpath, filename, arguments): pdb.DBBackedPackageHandle.__init__(packageDB, name, version, filelist) + self._relPath = relpath self._filename = filename self._arguments = arguments + def getRelativePath(self): + return self._relPath + def _doInstall(self): commandline = [ self._filename ] + self._arguments logging.info("Installing %s. Command line: %s", self._filename, diff --git a/lib/thandy/packagesys/PackageDB.py b/lib/thandy/packagesys/PackageDB.py index bb218be..c829ac3 100644 --- a/lib/thandy/packagesys/PackageDB.py +++ b/lib/thandy/packagesys/PackageDB.py @@ -1,34 +1,48 @@ # Copyright 2008 The Tor Project, Inc. See LICENSE for licensing information. import anydbm +import atexit import shelve import thandy.util import thandy.formats -class SimplePackageDB: +import thandy.packagesys.PackageSystem +class SimplePackageDB: def __init__(self, filename): + thandy.util.ensureParent(filename) self._db = anydbm.open(filename, 'c') + atexit.register(self.close) + + def close(self): + self._db.close() def setVersion(self, package, version, filelist): - pass + self._db['pv_%s'%package] = (version, filelist) def setInstallParameters(self, package, params): - pass + self._db['ip_%s'%package] = params def getCurVersion(self, package): - pass + v = self._db.get('pv_%s'%package) + if v != None: + return v[0] def getInstallParameters(self, package): - pass + return self._db.get('pi_%s'%package) +class DBBackedPackageSystem(thandy.packagesys.PackageSystem.PackageSystem): + def __init__(self): + self._packageDB = None -class DBBackedPackageSystem(thandy.packagesys.PackageSystem): - def __init__(self, packageDB): - self._packageDB = packageDB + def getDB(self): + if self._packageDB is None: + fname = thandy.util.userFilename("db/packages") + self._packageDB = pdb.PackageDB(fname) + return self._packageDB -class DBBackedPackageHandle(thandy.packagesys.PackageHandle): +class DBBackedPackageHandle(thandy.packagesys.PackageSystem.PackageHandle): def __init__(self, packageDB, name, version, filelist): thandy.packagesys.PackageSystem.PackageHandle.__init__(self) self._packageDB = packageDB @@ -68,14 +82,13 @@ class DBBackedPackageHandle(thandy.packagesys.PackageHandle): if not os.path.exists(fn): all_ok = False else: - f = open(fn, 'rb') try: - try: - d = thandy.formats.getFileDigest(f) - except OSError: + d = thandy.formats.getFileDigest(fn) + if d != hash: all_ok = False - break - finally: - f.close() + except OSError: + all_ok = False + break + return all_ok diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py index ad1e221..784de2d 100644 --- a/lib/thandy/packagesys/PackageSystem.py +++ b/lib/thandy/packagesys/PackageSystem.py @@ -1,10 +1,40 @@ # Copyright 2008 The Tor Project, Inc. See LICENSE for licensing information. +class PackageMetasystem: + def __init__(self, repository): + self._repostitory = repository + self._systems = {} + + def addPackageSystem(self, system): + self._systems[system.getName()] = system + + def getSysForPackage(self, pkg): + return self._systems.get(pkg['format'], None) + + @staticmethod + def create(repository): + r = PackageMetasystem(repository) + + try: + import rpm + except ImportError: + pass + else: + import thandy.packagesys.RPMPackages + r.addPackageSystem(thandy.packagesys.RPMPackages.RPMPackageSystem( + repository)) + + import thandy.packagesys.ExePackages + r.addPackageSystem(thandy.packagesys.ExePackages.ExePackageSystem( + repository)) + + return r + class PackageSystem: def getName(self): raise NotImplemented() - def packageHandleFromJSON(self, json): + def packageHandlesFromJSON(self, json): raise NotImplemented() def canBeAutomatic(self): @@ -39,6 +69,9 @@ class PackageTransaction: self._transactions.append(packageHandle.remove) class PackageHandle: + def getRelativePath(self): + raise NotImplemented() + def isInstalled(self, transaction=None): raise NotImplemented() diff --git a/lib/thandy/packagesys/RPMPackages.py b/lib/thandy/packagesys/RPMPackages.py index 5e4c960..4d0f326 100644 --- a/lib/thandy/packagesys/RPMPackages.py +++ b/lib/thandy/packagesys/RPMPackages.py @@ -6,14 +6,39 @@ import os import rpm import md5 +import thandy.formats + __all__ = [ 'RPMPackageSystem' ] class RPMPackageSystem(thandy.packagesys.PackageSystem.PackageSystem): + def __init__(self, repo): + self._repo = repo + def getName(self): - return "RPM" + return "rpm" + + def packageHandlesFromJSON(self, package): + if package['format'] != 'rpm': + raise thandy.FormatException() + + handles = [] + for entry in package['files']: + if len(entry) < 3: + continue + fn, h, extra = entry[:3] + name = os.path.split(fn)[1] - def packageHandleFromJSON(self, json): - raise NotImplemented() # XXXX + try: + version = extra['rpm_version'] + except KeyError: + raise thandy.FormatException() + + handles.append(RPMPackageHandle(name, + version, + fn, + self._repo.getFilename(fn))) + + return handles def getTransaction(self): return RPMPackageTransaction() @@ -115,7 +140,7 @@ def checkRPMInstall(name, version, ts=None): logging.info("%s is missing or unreadable from %s %s; " "that's ok.", fname, name, h['version']) else: - logging.warn("%s is missing or unreadable from %s %s." + logging.warn("%s is missing or unreadable from %s %s.", fname, name, h['version']) all_ok = False elif haveMD5 == md5sum: @@ -131,11 +156,15 @@ def checkRPMInstall(name, version, ts=None): return found and all_ok class RPMPackageHandle(thandy.packagesys.PackageSystem.PackageHandle): - def __init__(self, name, version, filename): + def __init__(self, name, version, relativePath, filename): self._name = name self._version = version + self._relPath = relativePath self._filename = filename + def getRelativePath(self): + return self._relPath + def anyVersionInstalled(self, transaction=None): return len(getInstalledRPMVersions(self.name, transaction)) > 1 diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 636ff86..696bbec 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -260,7 +260,7 @@ class LocalRepository: needRole='bundle') return pkg - def getRequestedFile(self, relPath): + def getRequestedFile(self, relPath, pkgSystems=None): """DOCDOC""" for f in self._metafiles: if f.getRelativePath() == relPath: @@ -279,12 +279,14 @@ class LocalRepository: return None - def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None): + def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None, + pkgSystems=None, installableDict=None): """Return a set of relative paths for all files that we need to fetch. Assumes that we care about the bundles 'trackingBundles'. If hashDict is provided, add mappings to it from the relative paths we want to fecth to the hashes that we want those items to have, when we know those hashes. + DOCDOC pkgSystems, installableDict """ if now == None: @@ -294,6 +296,9 @@ class LocalRepository: # Use a dummy hashdict. hashDict = {} + if installableDict == None: + installableDict = {} + need = set() # Fetch missing metafiles. @@ -444,8 +449,29 @@ class LocalRepository: # files? for pfile in packages.values(): package = pfile.get() + + alreadyInstalled = {} + allHandles = {} + if pkgSystems is not None: + psys = pkgSystems.getSysForPackage(package) + if psys is None: + logging.info("No way to check whether a %s package is " + "up-to-date." % package['format']) + else: + handles = psys.packageHandlesFromJSon(package) + + for h in handles: + allHandles[h.getRelativePath()] = h + if h.isInstalled(): + alreadyInstalled[h.getRelativePath()] = h + for f in package['files']: rp, h = f[:2] + if alreadyInstalled.has_key(rp): + logging.info("%s is already installed; no need to download", + rp) + continue + h_expected = thandy.formats.parseHash(h) hashDict[rp] = h_expected fn = self.getFilename(rp) @@ -459,6 +485,9 @@ class LocalRepository: if h_got != h_expected: logging.info("Hash for %s not as expected; must load.", rp) need.add(rp) + else: + if allHandles.has_key(rp): + installableDict[rp] = allHandles[rp] # Okay; these are the files we need. return need |