flumotion.common.bundle

1 # -*- Mode: Python; test-case-name: flumotion.test.test_common_bundle -*- 2 # vi:si:et:sw=4:sts=4:ts=4 3 # 4 # Flumotion - a streaming media server 5 # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 6 # All rights reserved. 7 8 # This file may be distributed and/or modified under the terms of 9 # the GNU General Public License version 2 as published by 10 # the Free Software Foundation. 11 # This file is distributed without any warranty; without even the implied 12 # warranty of merchantability or fitness for a particular purpose. 13 # See "LICENSE.GPL" in the source distribution for more information. 14 15 # Licensees having purchased or holding a valid Flumotion Advanced 16 # Streaming Server license may use this file in accordance with the 17 # Flumotion Advanced Streaming Server Commercial License Agreement. 18 # See "LICENSE.Flumotion" in the source distribution for more information. 19 20 # Headers in this file shall remain intact. 21 22 """ 23 bundles of files used to implement caching over the network 24 """ 25 26 import errno 27 import md5 28 import os 29 import zipfile 30 import tempfile 31 import StringIO 32 33 from flumotion.common import errors, dag 34 from flumotion.common.python import makedirs 35 36 __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket'] 37 __version__ = "$Rev: 6982 $" 38 39

40 -class BundledFile:

41 """ 42 I represent one file as managed by a bundler. 43 """

44 - def __init__(self, source, destination):

45 self.source = source 46 self.destination = destination 47 self._last_md5sum = None 48 self._last_timestamp = None 49 self.zipped = False

50

51 - def md5sum(self):

52 """ 53 Calculate the md5sum of the given file. 54 55 @returns: the md5 sum a 32 character string of hex characters. 56 """ 57 data = open(self.source, "r").read() 58 return md5.new(data).hexdigest()

59

60 - def timestamp(self):

61 """ 62 @returns: the last modified timestamp for the file. 63 """ 64 return os.path.getmtime(self.source)

65

66 - def hasChanged(self):

67 """ 68 Check if the file has changed since it was last checked. 69 70 @rtype: boolean 71 """ 72 73 # if it wasn't zipped yet, it needs zipping, so we pretend it 74 # was changed 75 # FIXME: move this out here 76 if not self.zipped: 77 return True 78 79 timestamp = self.timestamp() 80 # if file still has an old timestamp, it hasn't changed 81 # FIXME: looks bogus, shouldn't this check be != instead of <= ? 82 if self._last_timestamp and timestamp <= self._last_timestamp: 83 return False 84 self._last_timestamp = timestamp 85 86 # if the md5sum has changed, it has changed 87 md5sum = self.md5sum() 88 if self._last_md5sum != md5sum: 89 self._last_md5sum = md5sum 90 return True 91 92 return False

93

94 - def pack(self, zip):

95 self._last_timestamp = self.timestamp() 96 self._last_md5sum = self.md5sum() 97 zip.write(self.source, self.destination) 98 self.zipped = True

99

100 -class Bundle:

101 """ 102 I am a bundle of files, represented by a zip file and md5sum. 103 """

104 - def __init__(self, name):

105 self.zip = None 106 self.md5sum = None 107 self.name = name

108

109 - def setZip(self, zip):

110 """ 111 Set the bundle to the given data representation of the zip file. 112 """ 113 self.zip = zip 114 self.md5sum = md5.new(self.zip).hexdigest()

115

116 - def getZip(self):

117 """ 118 Get the bundle's zip data. 119 """ 120 return self.zip

121

122 -class Unbundler:

123 """ 124 I unbundle bundles by unpacking them in the given directory 125 under directories with the bundle's md5sum. 126 """

127 - def __init__(self, directory):

128 self._undir = directory

129

130 - def unbundlePathByInfo(self, name, md5sum):

131 """ 132 Return the full path where a bundle with the given name and md5sum 133 would be unbundled to. 134 """ 135 return os.path.join(self._undir, name, md5sum)

136

137 - def unbundlePath(self, bundle):

138 """ 139 Return the full path where this bundle will/would be unbundled to. 140 """ 141 return self.unbundlePathByInfo(bundle.name, bundle.md5sum)

142

143 - def unbundle(self, bundle):

144 """ 145 Unbundle the given bundle. 146 147 @type bundle: L{flumotion.common.bundle.Bundle} 148 149 @rtype: string 150 @returns: the full path to the directory where it was unpacked 151 """ 152 directory = self.unbundlePath(bundle) 153 154 filelike = StringIO.StringIO(bundle.getZip()) 155 zipFile = zipfile.ZipFile(filelike, "r") 156 zipFile.testzip() 157 158 filepaths = zipFile.namelist() 159 for filepath in filepaths: 160 path = os.path.join(directory, filepath) 161 parent = os.path.split(path)[0] 162 try: 163 makedirs(parent) 164 except OSError, err: 165 # Reraise error unless if it's an already existing 166 if err.errno != errno.EEXIST or not os.path.isdir(parent): 167 raise 168 data = zipFile.read(filepath) 169 170 # atomically write to path, see #373 171 fd, tempname = tempfile.mkstemp(dir=parent) 172 handle = os.fdopen(fd, 'wb') 173 handle.write(data) 174 handle.close() 175 176 # os.rename on Win32 is not deleting the target file 177 # if it exists, so remove it before 178 if os.path.exists(path): 179 os.unlink(path) 180 os.rename(tempname, path) 181 return directory

182

183 -class Bundler:

184 """ 185 I bundle files into a bundle so they can be cached remotely easily. 186 """

187 - def __init__(self, name):

188 """ 189 Create a new bundle. 190 """ 191 self._bundledFiles = {} # dictionary of BundledFile's indexed on path 192 self.name = name 193 self._bundle = Bundle(name)

194

195 - def add(self, source, destination = None):

196 """ 197 Add files to the bundle. 198 199 @param source: the path to the file to add to the bundle. 200 @param destination: a relative path to store this file in in the bundle. 201 If unspecified, this will be stored in the top level. 202 203 @returns: the path the file got stored as 204 """ 205 if destination == None: 206 destination = os.path.split(source)[1] 207 self._bundledFiles[source] = BundledFile(source, destination) 208 return destination

209

210 - def bundle(self):

211 """ 212 Bundle the files registered with the bundler. 213 214 @rtype: L{flumotion.common.bundle.Bundle} 215 """ 216 # rescan files registered in the bundle, and check if we need to 217 # rebuild the internal zip 218 if not self._bundle.getZip(): 219 self._bundle.setZip(self._buildzip()) 220 return self._bundle 221 222 update = False 223 for bundledFile in self._bundledFiles.values(): 224 if bundledFile.hasChanged(): 225 update = True 226 break 227 228 if update: 229 self._bundle.setZip(self._buildzip()) 230 231 return self._bundle

232 233 # build the zip file containing the files registered in the bundle 234 # and return the zip file data

235 - def _buildzip(self):

236 filelike = StringIO.StringIO() 237 zipFile = zipfile.ZipFile(filelike, "w") 238 for bundledFile in self._bundledFiles.values(): 239 bundledFile.pack(zipFile) 240 zipFile.close() 241 data = filelike.getvalue() 242 filelike.close() 243 return data

244

245 -class BundlerBasket:

246 """ 247 I manage bundlers that are registered through me. 248 """

249 - def __init__(self):

250 """ 251 Create a new bundler basket. 252 """ 253 self._bundlers = {} # bundler name -> bundle 254 255 self._files = {} # filename -> bundle name 256 self._imports = {} # import statements -> bundle name 257 258 self._graph = dag.DAG()

259

260 - def add(self, bundleName, source, destination=None):

261 """ 262 Add files to the bundler basket for the given bundle. 263 264 @param bundleName: the name of the bundle this file is a part of 265 @param source: the path to the file to add to the bundle 266 @param destination: a relative path to store this file in in the bundle. 267 If unspecified, this will be stored in the top level 268 """ 269 # get the bundler and create it if need be 270 if not bundleName in self._bundlers: 271 bundler = Bundler(bundleName) 272 self._bundlers[bundleName] = bundler 273 else: 274 bundler = self._bundlers[bundleName] 275 276 # add the file to the bundle and register 277 location = bundler.add(source, destination) 278 if location in self._files: 279 raise Exception("Cannot add %s to bundle %s, already in %s" % ( 280 location, bundleName, self._files[location])) 281 self._files[location] = bundleName 282 283 # add possible imports from this file 284 package = None 285 if location.endswith('.py'): 286 package = location[:-3] 287 elif location.endswith('.pyc'): 288 package = location[:-4] 289 290 if package: 291 if package.endswith('__init__'): 292 package = os.path.split(package)[0] 293 294 package = ".".join(package.split('/')) # win32 fixme 295 if package in self._imports: 296 raise Exception("Bundler %s already has import %s" % ( 297 bundleName, package)) 298 self._imports[package] = bundleName

299

300 - def depend(self, depender, *dependencies):

301 """ 302 Make the given bundle depend on the other given bundles. 303 304 @type depender: string 305 @type dependencies: list of strings 306 """ 307 # note that a bundler doesn't necessarily need to be registered yet 308 if not self._graph.hasNode(depender): 309 self._graph.addNode(depender) 310 for dep in dependencies: 311 if not self._graph.hasNode(dep): 312 self._graph.addNode(dep) 313 self._graph.addEdge(depender, dep)

314

315 - def getDependencies(self, bundlerName):

316 """ 317 Return names of all the dependencies of this bundle, including this 318 bundle itself. 319 The dependencies are returned in a correct depending order. 320 """ 321 if not bundlerName in self._bundlers: 322 raise errors.NoBundleError('Unknown bundle %s' % bundlerName) 323 elif not self._graph.hasNode(bundlerName): 324 return [bundlerName] 325 else: 326 return [bundlerName] + self._graph.getOffspring(bundlerName)

327

328 - def getBundlerByName(self, bundlerName):

329 """ 330 Return the bundle by name, or None if not found. 331 """ 332 if self._bundlers.has_key(bundlerName): 333 return self._bundlers[bundlerName] 334 return None

335

336 - def getBundlerNameByImport(self, importString):

337 """ 338 Return the bundler name by import statement, or None if not found. 339 """ 340 if self._imports.has_key(importString): 341 return self._imports[importString] 342 return None

343

344 - def getBundlerNameByFile(self, filename):

345 """ 346 Return the bundler name by filename, or None if not found. 347 """ 348 if self._files.has_key(filename): 349 return self._files[filename] 350 return None

351

352 - def getBundlerNames(self):

353 """ 354 Get all bundler names. 355 356 @rtype: list of str 357 @returns: a list of all bundler names in this basket. 358 """ 359 return self._bundlers.keys()

360

361 -class MergedBundler(Bundler):

362 """ 363 I am a bundler, with the extension that I can also bundle other 364 bundlers. 365 366 The effect is that when you call bundle() on a me, you get one 367 bundle with a union of all subbundlers' files, in addition to any 368 loose files that you added to me. 369 """

370 - def __init__(self, name='merged-bundle'):

371 Bundler.__init__(self, name) 372 self._subbundlers = {}

373

374 - def addBundler(self, bundler):

375 """Add to me all of the files managed by another bundler. 376 377 @param bundler: The bundler whose files you want in this 378 bundler. 379 @type bundler: L{Bundler} 380 """ 381 if bundler.name not in self._subbundlers: 382 self._subbundlers[bundler.name] = bundler 383 for bfile in bundler._files.values(): 384 self.add(bfile.source, bfile.destination)

385

386 - def getSubBundlers(self):

387 """ 388 @returns: A list of all of the bundlers that have been added to 389 me. 390 """ 391 return self._subbundlers.values()

392

Source Code for Module flumotion.common.bundle