From 3bef6462fa8117dde2f55a3ff06dbf1b1a8e02a8 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Sun, 30 Nov 2008 06:19:04 +0000 Subject: Big thandy installation refactoring. Things should be saner now: we recognize that checking an item is sometimes orthogonal to installing it; we do not carry around big bits of unimplemented machinery; we actually document what stuff does in thandy-spec.txt; we do more OO in the places that make sense and less in the places that do not; and almost as an afterthought, we support the command installer type internally. Now all we need is a frontend to make command installers. git-svn-id: file:///home/or/svnrepo/updater/trunk@17414 55e972cd-5a19-0410-ae62-a4d7a52db4cd --- lib/thandy/packagesys/ExePackages.py | 145 ++++++++++------------ lib/thandy/packagesys/PackageDB.py | 153 +++++++++++++++-------- lib/thandy/packagesys/PackageSystem.py | 218 ++++++++++++++++++++++----------- lib/thandy/packagesys/RPMPackages.py | 137 +++++++++------------ lib/thandy/packagesys/__init__.py | 3 + 5 files changed, 376 insertions(+), 280 deletions(-) (limited to 'lib/thandy/packagesys') diff --git a/lib/thandy/packagesys/ExePackages.py b/lib/thandy/packagesys/ExePackages.py index be0517a..59be63e 100644 --- a/lib/thandy/packagesys/ExePackages.py +++ b/lib/thandy/packagesys/ExePackages.py @@ -2,84 +2,75 @@ import subprocess import logging +import re +import os import thandy.util -import thandy.packagesys.PackageSystem as ps -import thandy.packagesys.PackageDB as pdb +import thandy.packagesys.PackageSystem as PS +import thandy.packagesys.PackageDB as PDB + +class RegistryChecker(PS.Checker): + def __init__(self, key, version): + PS.Checker.__init__(self) + self._key = key + self._version = version + + def __repr__(self): + return "RegistryChecker(%r, %r)"%(self._key, self._version) + + def getInstalledVersions(self): + try: + return [ thandy.util.getRegistryValue(self._key) ] + except thandy.util.NoRegistry: + raise thandy.CheckNotSupported("This OS has no registry.") + + def isInstalled(self): + return self._version in self.getInstalledVersions() + +class CommandInstaller(PS.Installer): + def __init__(self, relPath, installCommand, removeCommand=None): + PS.Installer.__init__(self, relPath) + self._installCommand = installCommand + self._removeCommand = removeCommand + + def __repr__(self): + parts = [ "CommandInstaller(%r, %r" %(self._relPath, + self._installCommand) ] + if self.removeCommand: + parts.append(", %r"%self.removeCommand) + parts.append(")") + return "".join(parts) + + def install(self): + self._runCommand(self._installCommand) + + def remove(self): + if self._removeCommand: + raise thandy.RemoveNotSupported() + self._runCommand(self._removeCommand) + + def _runCommand(self, command): + d = { "FILE": self.getFilename() } + def replace(m): + return d[m.group(1)] + try: + c = [ re.sub(r'\$\{([\w_]+)\}', replace, word) for word in command ] + except KeyError: + raise thandy.InstallFailed("Unrecognized option in command %s" + %command) + logging.info("Installing %s. Command is %s", self._relPath, c) + + return_code = self._execute(c) + if return_code != 0: + raise thandy.InstallFailed("Return code %s from calling %s"% + (return_code, c)) + + def _execute(self, cmd): + try: + return subprocess.call(cmd) + except OSError, e: + logging.warn("Error from trying to call %s: %s", cmd, e) + raise thandy.InstallFailed("Could not execute install command %s" + %cmd) -class ExePackageSystem(pdb.DBBackedPackageSystem): - def __init__(self, repo): - pdb.DBBackedPackageSystem.__init__(self) - self._repo = repo - - def getName(self): - return "exe" - - def packageHandlesFromJSON(self, pkg): - if pkg['format'] != 'exe': - raise thandy.FormatException() - - handles = [] - for entry in pkg['files']: - if len(entry) < 3: - continue - rp, h, extra = entry[:3] - version = pkg['version'] - - handles.append( - ExePackageHandle(self.getDB(), - pkg['name'], - version, - [], # filelist not implemented in this. - rp, - self._repo.getFilename(rp), - arguments=extra.get('exe_args', []), - registry_ent=extra.get('registry_ent'))) - return handles - - def canBeAutomatic(self): - return True - - def canHaveUI(self): - return True - -class ExePackageHandle(pdb.DBBackedPackageHandle): - def __init__(self, packageDB, name, version, filelist, relpath, filename, - arguments, registry_ent=None): - pdb.DBBackedPackageHandle.__init__(self, packageDB, name, version, filelist) - self._relPath = relpath - self._filename = filename - self._arguments = arguments - self._registry_ent = registry_ent - - def getRelativePath(self): - return self._relPath - - def getInstalledVersion(self, transaction=None): - if self._registry_ent != None: - try: - ver = thandy.util.getRegistryValue(self._registry_ent[0]) - if ver != None: - return ver - except thandy.util.NoRegistry: - pass - - return pdb.DBBackedPackageHandle.getInstalledVersion(self, transaction) - - def isInstalled(self, transaction=None): - if self._registry_ent != None: - try: - ver = thandy.util.getRegistryValue(self._registry_ent[0]) - return ver == self._registry_ent[1] - except thandy.util.NoRegistry: - pass - - return pdb.DBBackedPackageHandle.isInstalled(self, transaction) - - - def _doInstall(self): - commandline = [ self._filename ] + self._arguments - logging.info("Installing %s. Command line: %s", self._filename, - commandline) - subprocess.call(commandline) diff --git a/lib/thandy/packagesys/PackageDB.py b/lib/thandy/packagesys/PackageDB.py index a81507d..94a5ad9 100644 --- a/lib/thandy/packagesys/PackageDB.py +++ b/lib/thandy/packagesys/PackageDB.py @@ -2,13 +2,18 @@ import atexit import shelve +import logging import thandy.util import thandy.formats -import thandy.packagesys.PackageSystem +import thandy.packagesys.PackageSystem as PS class SimplePackageDB: + """Trivial wrapper around Python's shelve module to provide storage for + installation information for package items that don't automatically + record their presence. + """ def __init__(self, filename): thandy.util.ensureParentDir(filename) self._db = shelve.open(filename, 'c') @@ -23,71 +28,121 @@ class SimplePackageDB: def setInstallParameters(self, package, params): self._db['ip_%s'%str(package)] = params + def setManifest(self, package, fnameToDigest): + self._db['mf_%s'%str(package)] = fnameToDigest + def getCurVersion(self, package): v = self._db.get('pv_%s'%str(package)) if v != None: return v[0] + else: + return None def getInstallParameters(self, package): return self._db.get('pi_%s'%str(package)) -class DBBackedPackageSystem(thandy.packagesys.PackageSystem.PackageSystem): - def __init__(self): - self._packageDB = None + def getManifest(self, package): + return self._db.get('mf_%'%str(package), {}) - def getDB(self): - if self._packageDB is None: - fname = thandy.util.userFilename("db/packages") - self._packageDB = SimplePackageDB(fname) - return self._packageDB - -class DBBackedPackageHandle(thandy.packagesys.PackageSystem.PackageHandle): - def __init__(self, packageDB, name, version, filelist): - thandy.packagesys.PackageSystem.PackageHandle.__init__(self) - self._packageDB = packageDB - self._name = name - self._version = version - self._filelist = filelist + def removeAll(self, package): + for template in ["pv_%s", "ip_%s", "mf_%s"]: + try: + del self._db[template % str(package)] + except KeyError: + pass + +_DB_INSTANCE = None - self._metaData = None +def getPackageDBInstance(): + global _DB_INSTANCE + if _DB_INSTANCE == None: + fname = thandy.util.userFilename("db/packages") + logging.info("Opening package database in %s", fname) + _DB_INSTANCE = SimplePackageDB(fname) + return _DB_INSTANCEx - def _getInstallBase(self): - raise NotImplemented() +class _DBMixin: + def setDB(self, db): + self._db = db - def anyVersionInstalled(self, transaction=None): - return self.getInstalledVersion(transaction) != None + def getDB(self): + if self._db is None: + self._db = getPackageDBInstance() + return self._db + +class DBChecker(PS.Checker, _DBMixin): + def __init__(self, name, version): + PS.Checker.__init__(self) + self._name = name + self._version = version + self._db = None + + def __repr__(self): + return "DBChecker(%r, %r)"%(self._name, self._version) + +# def checkInstall(self): +# if not self.isInstalled(): +# return False +# else: +# return self._checkManifest() +# +# def _getInstallRoot(self): +# return "/" +# +# def _checkManifest(self): +# manifest = self.getDB().getManifest(self._name) +# root = self._getInstallRoot() +# all_ok = True +# for fname, digest_want in manifest: +# real_fname = os.path.join(self._getInstallRoot(), fname) +# logging.info("Checking digest on %s", fname) +# try: +# digest = thandy.formats.getFileDigest(real_fname): +# if digest != digest_want: +# logging.warn("Digest on %s not as expected", real_fname) +# all_ok = False +# except OSError: +# logging.warn("File %s not found.", real_fname) +# all_ok = False +# return all_ok +# + def getInstalledVersions(self): + return [ self.getDB().getCurVersion(self._name) ] + + def isInstalled(self): + return self._version in self.getInstalledVersions(transaction) + +class DBInstaller(PS.Installer, _DBMixin): + def __init__(self, name, version, relPath, installer): + PS.Installer.__init__(self, relPath) + self._name = name + self._version = version + self._installer = installer - def getInstalledVersion(self, transaction=None): - return self._packageDB.getCurVersion(self._name) + def __repr__(self): + return "DBInstaller(%r, %r, %r, %r)"%(self._name, + self._version, + self._relPath, + self._installer) - def install(self, transaction=None): - params = self._doInstall() - self._packageDB.setVersion( - self._name, self._version, self._filelist) - self._packageDB.setInstallParameters(self._name, params) + def setTransaction(self, transaction): + self._installer.setTransaction(transaction) - def _doInstall(self): - raise NotImplemented() + def setCacheRoot(self, cacheRoot): + self._installer.setCacheRoot(cacheRoot) - def isInstalled(self, transaction=None): - return self.getInstalledVersion(transaction) == self._version + def install(self): + self._installer.install() - def checkInstall(self, transaction=None): - base = self._getInstallBase() + params, manifest = self._installer.getInstallResult() + self.getDB().setCurVersion(self._name, self._version) + if params != None: + self.getDB().getInstallParameters(self._name, params) + if manifest != None: + self.getDB().setManifest(self._name, manifest) - all_ok = True - for fn, hash in self._filelist: - fn = os.path.join(base, fn) - if not os.path.exists(fn): - all_ok = False - else: - try: - d = thandy.formats.getFileDigest(fn) - if d != hash: - all_ok = False - except OSError: - all_ok = False - break + def remove(self): + self._installer.remove() + self.getDB().removeAll(self._name) - return all_ok diff --git a/lib/thandy/packagesys/PackageSystem.py b/lib/thandy/packagesys/PackageSystem.py index fa0d28b..4a847e9 100644 --- a/lib/thandy/packagesys/PackageSystem.py +++ b/lib/thandy/packagesys/PackageSystem.py @@ -1,94 +1,166 @@ # Copyright 2008 The Tor Project, Inc. See LICENSE for licensing information. -class PackageMetasystem: - def __init__(self, repository): - self._repostitory = repository - self._systems = {} - - def addPackageSystem(self, system): - self._systems[system.getName()] = system - - def getSysForPackage(self, pkg): - return self._systems.get(pkg['format'], None) - - @staticmethod - def create(repository): - r = PackageMetasystem(repository) - - try: - import rpm - except ImportError: - pass +import os + +def getItemsFromPackage(pkg): + result = {} + format = pkg.get('format') + for item in pkg['files']: + relPath = item[0] + if len(item) >= 3: + extra = item[2] else: + extra = {} + checkFormat = extra.get("check_type") + installFormat = extra.get("install_type") + + checker = getChecker(checkFormat, relPath, extra, defaultFormat=format, + package=pkg) + installer = getInstaller(installFormat, relPath, extra, + defaultFormat=format, package=pkg) + result[relPath] = PackageItem(relPath, checker, installer) + return result + +def getChecker(checkType, relPath, extra, defaultFormat, package): + if checkType == None: + #DOCDOC obsolete + if defaultFormat == 'rpm': import thandy.packagesys.RPMPackages - r.addPackageSystem(thandy.packagesys.RPMPackages.RPMPackageSystem( - repository)) - + return thandy.packagesys.RPMPackages.RPMChecker( + os.path.split(relPath)[1], + extra['rpm_version']) + elif defaultFormat == 'exe': + if extra.has_key('registry_ent'): + import thandy.packagesys.ExePackages + k,v=extra['registry_ent'] + return thandy.packagesys.ExePackages.RegistryChecker(k, v) + else: + import thandy.packagesys.PackageDB + return thandy.packagesys.PackageDB.DBChecker( + package['name'], package['version']) + else: + return None + elif checkType == 'rpm': + import thandy.packagesys.RPMPackages + return thandy.packagesys.RPMPackages.RPMChecker( + os.path.split(relPath)[1], + extra['rpm_version']) + elif checkType == 'db': + import thandy.packagesys.PackageDB + return thandy.packagesys.PackageDB.DBChecker( + extra['item_name'], extra['item_version']) + elif checkType == 'registry': import thandy.packagesys.ExePackages - r.addPackageSystem(thandy.packagesys.ExePackages.ExePackageSystem( - repository)) - - return r - -class PackageSystem: - def getName(self): - raise NotImplemented() - - def packageHandlesFromJSON(self, json): - raise NotImplemented() - - def canBeAutomatic(self): - return True - - def canHaveUI(self): - return False - - def getTransaction(self): - return PackageTransaction() - -class PackageTransaction: + k,v=extra['registry_ent'] + return thandy.packagesys.ExePackages.RegistryChecker(k,v) + else: + return None + +def getInstaller(installType, relPath, extra, defaultFormat, package): + if installType == None: + # XXX obsolete. + if defaultFormat == 'rpm': + import thandy.packagesys.RPMPackages + return thandy.packagesys.RPMPackages.RPMInstaller( + relPath, os.path.split(relPath)[1]) + elif defaultFormat == 'exe': + import thandy.packagesys.ExePackages + installer = thandy.packagesys.ExePackages.CommandInstaller( + relPath, [ "${FILE}" ] + extra.get('exe_args', [])) + if not extra.has_key('registry_ent'): + import thandy.packagesys.PackageDB + installer = thandy.packagesys.PackageDB.DBInstaller( + package['name'], package['version'], relPath, installer) + return installer + else: + return None + elif installType == 'rpm': + import thandy.packagesys.RPMPackages + installer = thandy.packagesys.RPMPackages.RPMInstaller( + relPath, os.path.split(relPath)[1]) + elif installType == 'command': + import thandy.packagesys.ExePackages + installer = thandy.packagesys.ExePackages.CommandInstaller( + relPath, extra['cmd_install'], extra.get['cmd_remove']) + else: + return None + + if extra.get('check_type') == 'db': + import thandy.packagesys.PackageDB + installer = thandy.packagesys.PackageDB.DBInstaller( + extra['item_name'], extra['item_version'], installer) + + return installer + +class PackageItem: + def __init__(self, relativePath, checker, installer): + self._relPath = relativePath + self._checker = checker + self._installer = installer + + def setTransaction(self, transaction): + if self._cheker is not None: + self._checker.setTransaction(transaction) + if self._installer is not None: + self._installer.setTransaction(transaction) + def setCacheRoot(self, cacheRoot): + if self._installer is not None: + self._installer.setCacheRoot(cacheRoot) + + def canCheck(self): + return self._checker != None + def canInstall(self): + return self._installer != None + def getChecker(self): + return self._checker + def getInstaller(self): + return self._installer + +class Checker: def __init__(self): - self._transactions = [] + self._transaction = None - def _start(self): - pass + def setTransaction(self, transaction): + self._transaction = transaction - def _commit(self): - pass +# def checkInstall(self): +# raise NotImplemented() - def run(self): - self._start() - for cb in self._transactions: - cb(self) - self._commit() + def anyVersionInstalled(self): + raise len(self.getInstalledVersions()) > 1 - def addInstall(self, packageHandle): - self._transactions.append(packageHandle.install) + def getInstalledVersions(self): + raise NotImplemented() - def addRemove(self, packageHandle): - self._transactions.append(packageHandle.remove) + def isInstalled(self): + raise NotImplemented() -class PackageHandle: - def __init__(self): - pass +class Installer: + def __init__(self, relativePath): + self._transaction = None + self._cacheRoot = None + self._relPath = relativePath - def getRelativePath(self): - raise NotImplemented() + def setTransaction(self, transaction): + self._transaction = transaction - def isInstalled(self, transaction=None): - raise NotImplemented() + def setCacheRoot(self, cacheRoot): + self._cacheRoot = cacheRoot - def anyVersionInstalled(self, transaction=None): - raise NotImplemented() + def getFilename(self): + rp = self._relPath + if rp.startswith('/'): + rp = rp[1:] + return os.path.normpath(os.path.join(self._cacheRoot, rp)) - def getInstalledVersion(self, transaction=None): + def install(self, relativePath, root): raise NotImplemented() - def install(self, transaction): + def remove(self): raise NotImplemented() - def remove(self, transaction): - raise NotImplemented() + def getInstallResult(self): + "DOCDOC params, manifest" + return None, None + - def checkInstall(self, transaction=None): - raise NotImplemented() diff --git a/lib/thandy/packagesys/RPMPackages.py b/lib/thandy/packagesys/RPMPackages.py index 4d0f326..c710b8d 100644 --- a/lib/thandy/packagesys/RPMPackages.py +++ b/lib/thandy/packagesys/RPMPackages.py @@ -1,48 +1,18 @@ # Copyright 2008 The Tor Project, Inc. See LICENSE for licensing information. -import thandy.packagesys.PackageSystem +import thandy.packagesys.PackageSystem as PS import os -import rpm +try: + import rpm +except ImportError: + rpm = None import md5 import thandy.formats __all__ = [ 'RPMPackageSystem' ] -class RPMPackageSystem(thandy.packagesys.PackageSystem.PackageSystem): - def __init__(self, repo): - self._repo = repo - - def getName(self): - return "rpm" - - def packageHandlesFromJSON(self, package): - if package['format'] != 'rpm': - raise thandy.FormatException() - - handles = [] - for entry in package['files']: - if len(entry) < 3: - continue - fn, h, extra = entry[:3] - name = os.path.split(fn)[1] - - try: - version = extra['rpm_version'] - except KeyError: - raise thandy.FormatException() - - handles.append(RPMPackageHandle(name, - version, - fn, - self._repo.getFilename(fn))) - - return handles - - def getTransaction(self): - return RPMPackageTransaction() - _CALLBACK_CODES = {} for name in dir(rpm): @@ -50,33 +20,33 @@ for name in dir(rpm): _CALLBACK_CODES[getattr(rpm, name)] = name[12:] del name -class RPMPackageTransaction(thandy.packagesys.PackageSystem.PackageTransaction): - - def _start(self): - thandy.packagesys.PackageSystem.PackageTransaction.__init__(self) - self._tset = rpm.TransactionSet() - - def _commit(self): - self._tset.run(self._callback, "") - - def _callback(self, what, amount, total, mydata, _): - if what == rpm.RPMCALLBACK_INST_OPEN_FILE: - hdr, path = mydata - logging.info("Installing RPM for %s [%s]", hdr['name'], path) - - elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE: - hdr, path = mydata - logging.info("Done installing RPM for %s", path) - - elif what == rpm.RPMCALLBACK_INST_PROGRESS: - hdr, path = mydata - logging.info("%s: %.5s%% done", name, float(amount)/total*100) - - else: - hdr, path = mydata - logging.info("RPM event %s on %s [%s/%s]", - _CALLBACK_CODES.get(what,str(what)), - hdr['name'], amount, total) +class RPMPackageTransaction: + + def _start(self): + PS.PackageTransaction.__init__(self) + self._tset = rpm.TransactionSet() + + def _commit(self): + self._tset.run(self._callback, "") + + def _callback(self, what, amount, total, mydata, _): + if what == rpm.RPMCALLBACK_INST_OPEN_FILE: + hdr, path = mydata + logging.info("Installing RPM for %s [%s]", hdr['name'], path) + + elif what == rpm.RPMCALLBACK_INST_CLOSE_FILE: + hdr, path = mydata + logging.info("Done installing RPM for %s", path) + + elif what == rpm.RPMCALLBACK_INST_PROGRESS: + hdr, path = mydata + logging.info("%s: %.5s%% done", name, float(amount)/total*100) + + else: + hdr, path = mydata + logging.info("RPM event %s on %s [%s/%s]", + _CALLBACK_CODES.get(what,str(what)), + hdr['name'], amount, total) def addRPMInstall(ts, path): fd = os.open(path, os.O_RDONLY) @@ -155,31 +125,36 @@ def checkRPMInstall(name, version, ts=None): return found and all_ok -class RPMPackageHandle(thandy.packagesys.PackageSystem.PackageHandle): - def __init__(self, name, version, relativePath, filename): - self._name = name - self._version = version - self._relPath = relativePath - self._filename = filename +class RPMChacker(PS.Checker): + def __init__(self, rpmname, rpmversion): + PS.Checker.__init__(self) + self._name = rpmname + self._version = rpmversion + + def __repr__(self): + return "RPMChecker(%r, %r)"%(self._name, self._version) - def getRelativePath(self): - return self._relPath + def getInstalledVersions(self): + return getInstalledRPMVersions(self._name, self._transaction) - def anyVersionInstalled(self, transaction=None): - return len(getInstalledRPMVersions(self.name, transaction)) > 1 + def isInstalled(self): + vers = getInstalledRPMVersions(self._name, self._transaction) + return self._version in vers - def getInstalledVersion(self, transaction=None): - s = max(getInstalledRPMVersions(self._name, transaction)) +# def checkInstall(self): +# return checkRPMInstall(self._name, self._version) + +class RPMInstaller(PS.Installer): + def __init__(self, rpmname, relPath): + PS.Installer.__init__(self, relPath) + self._name = rpmname + + def __repr__(self): + return "RPMInstaller(%r, %r)"%(self._name, self._relPath) def install(self, transaction): - addRPMInstall(transaction._trans, self._filename) + addRPMInstall(transaction._trans, self.getFilename()) def remove(self, transaction): addRPMErase(transaction._trans, self._name) - def isInstalled(self, transaction=None): - return self._version in getInstalledRPMVersions(self._name,transaction) - - def checkInstall(self, transaction=None): - return checkRPMInstall(self._name, self._version) - diff --git a/lib/thandy/packagesys/__init__.py b/lib/thandy/packagesys/__init__.py index 4dd7019..dc0825b 100644 --- a/lib/thandy/packagesys/__init__.py +++ b/lib/thandy/packagesys/__init__.py @@ -2,3 +2,6 @@ __all__ = [ ] + + + -- cgit v1.2.3