From 1305f1225085f8e3ba9febfbc2e9895a91d4a510 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 17 Nov 2008 00:28:22 +0000 Subject: make log levels configurable. check file hashes correctly. git-svn-id: file:///home/or/svnrepo/updater/trunk@17301 55e972cd-5a19-0410-ae62-a4d7a52db4cd --- lib/thandy/ClientCLI.py | 32 ++++++++++++++++++++++---------- lib/thandy/download.py | 14 +++++++++----- lib/thandy/repository.py | 34 +++++++++++++++++++++++----------- 3 files changed, 54 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/thandy/ClientCLI.py b/lib/thandy/ClientCLI.py index 9516d1a..05f3647 100644 --- a/lib/thandy/ClientCLI.py +++ b/lib/thandy/ClientCLI.py @@ -18,12 +18,15 @@ def update(args): repoRoot = thandy.util.userFilename("cache") options, args = getopt.getopt(args, "", [ "repo=", "no-download", "loop", "no-packagesys", - "install", "socks-port="]) + "install", "socks-port=", + "debug", "info", + "warn"]) download = True keep_looping = False use_packagesys = True install = False socksPort = None + logLevel = logging.INFO for o, v in options: if o == '--repo': @@ -38,6 +41,14 @@ def update(args): install = True elif o == "--socks-port": socksPort = int(v) + elif o == '--debug': + logLevel = logging.DEBUG + elif o == '--info': + logLevel = logging.INFO + elif o == '--warn': + logLevel = logging.WARN + + logging.basicConfig(level=logLevel) if socksPort: thandy.socksurls.setSocksProxy("127.0.0.1", socksPort) @@ -59,7 +70,7 @@ def update(args): installableDict=installable) if installable and not files: - logging.notice("Ready to install files: %s", + logging.info("Ready to install files: %s", ", ".join(sorted(installable.keys()))) if install: # XXXX handle ordering @@ -95,10 +106,12 @@ def update(args): downloader = thandy.download.DownloadManager() for f in files: - dj = thandy.download.ThandyDownloadJob(f, repo.getFilename(f), - mirrorlist, - wantHash=hashes.get(f), - useTor=(socksPort!=None)) + dj = thandy.download.ThandyDownloadJob( + f, repo.getFilename(f), + mirrorlist, + wantHash=hashes.get(f), + repoFile=repo.getRequestedFile(f), + useTor=(socksPort!=None)) def successCb(rp=f): rf = repo.getRequestedFile(rp) @@ -108,10 +121,10 @@ def update(args): downloader.addDownloadJob(dj) - logging.info("Launching downloads") + logging.debug("Launching downloads") downloader.start() - logging.info("Waiting for downloads to finish.") + logging.debug("Waiting for downloads to finish.") downloader.wait() logging.info("All downloads finished.") @@ -123,11 +136,10 @@ def usage(): print "Known commands:" print " update [--repo=repository] [--no-download] [--loop]" print " [--no-packagesys] [--install] [--socks-port=port]" + print " [--debug|--info|--warn]" sys.exit(1) def main(): - #XXXX make this an option. - logging.basicConfig(level=logging.DEBUG) if len(sys.argv) < 2: usage() diff --git a/lib/thandy/download.py b/lib/thandy/download.py index aaa8215..dab9a2c 100644 --- a/lib/thandy/download.py +++ b/lib/thandy/download.py @@ -130,16 +130,18 @@ class DownloadManager: class DownloadJob: """Abstract base class. Represents a thing to be downloaded, and the knowledge of how to download it.""" - def __init__(self, targetPath, tmpPath, wantHash=None, useTor=False): + def __init__(self, targetPath, tmpPath, wantHash=None, repoFile=None, + useTor=False): """Create a new DownloadJob. When it is finally downloaded, store it in targetPath. Store partial results in tmpPath; if there is already a file in tmpPath, assume that it is an incomplete download. If wantHash, reject the file unless the hash is as given. If useTor, use a socks connection.""" - + #DOCDODC repofile self._destPath = targetPath self._tmpPath = tmpPath self._wantHash = wantHash + self._repoFile = repoFile self._useTor = useTor self._success = lambda : None @@ -235,10 +237,12 @@ class DownloadJob: if f_out is not None: f_out.close() - if self._wantHash: + if self._wantHash and not self._repoFile: gotHash = thandy.formats.getFileDigest(self._tmpPath) if gotHash != self._wantHash: raise thandy.DownloadError("File hash was not as expected.") + elif self._repoFile: + self._repoFile.checkFile(self._tmpPath, self._wantHash) thandy.util.ensureParentDir(self._destPath) thandy.util.moveFile(self._tmpPath, self._destPath) @@ -264,10 +268,10 @@ class ThandyDownloadJob(DownloadJob): """Thandy's subtype of DownloadJob: knows about mirrors, weighting, and Thandy's directory structure.""" def __init__(self, relPath, destPath, mirrorList, wantHash=None, - supportedURLTypes=None, useTor=None): + supportedURLTypes=None, useTor=None, repoFile=None): DownloadJob.__init__(self, destPath, None, wantHash=wantHash, - useTor=useTor) + useTor=useTor, repoFile=repoFile) self._mirrorList = mirrorList self._relPath = relPath diff --git a/lib/thandy/repository.py b/lib/thandy/repository.py index 3e1f441..6378586 100644 --- a/lib/thandy/repository.py +++ b/lib/thandy/repository.py @@ -128,6 +128,20 @@ class RepositoryFile: return signed_obj, main_obj + def checkFile(self, fname, needhash=None): + f = open(fname, 'r') + try: + s = f.read() + finally: + f.close() + + signed, main = self._checkContent(s) + if needhash: + d = thandy.formats.getDigest(main) + if d != needhash: + raise thandy.FormatException("Content didn't match needed " + "hash.") + def load(self): """Load this object from disk if it hasn't already been loaded.""" if self._main_obj == None: @@ -186,8 +200,10 @@ class PkgFile: def getExpectedHash(self): return self._needHash - def checkFile(self): - return self._needHash == self._repository.getFileDigest() + def checkFile(self, fname, needHash=None): + if needHash: + if thandy.formats.getFileDigest(fname) != needHash: + raise thandy.FormatException("Digest for %s not as expected.") class LocalRepository: """Represents a client's partial copy of a remote mirrored repository.""" @@ -262,7 +278,7 @@ class LocalRepository: def getRequestedFile(self, relPath, pkgSystems=None): """DOCDOC""" - for f in self._metafiles: + for f in self._metaFiles: if f.getRelativePath() == relPath: return f for f in self._bundleFiles.itervalues(): @@ -283,10 +299,8 @@ class LocalRepository: pkgSystems=None, installableDict=None): """Return a set of relative paths for all files that we need to fetch. Assumes that we care about the bundles - 'trackingBundles'. If hashDict is provided, add mappings to it - from the relative paths we want to fecth to the hashes that we - want those items to have, when we know those hashes. - DOCDOC pkgSystems, installableDict + 'trackingBundles'. + DOCDOC pkgSystems, installableDict, hashDict """ if now == None: @@ -390,9 +404,8 @@ class LocalRepository: continue rp = binfo.getRelativePath() - #hashDict[rp] = #XXXX this hash needs to be calculated ovver - # # the json data. h_expected = binfo.getHash() + hashDict[rp] = h_expected bfile = self.getBundleFile(rp) try: bfile.load() @@ -425,8 +438,7 @@ class LocalRepository: rp = pkginfo['path'] pfile = self.getPackageFile(rp) h_expected = thandy.formats.parseHash(pkginfo['hash']) - #hashDict[rp] = #XXXX this hash needs to be calculated ovver - # # the json data. + hashDict[rp] = h_expected try: pfile.load() except OSError: -- cgit v1.2.3