Package flumotion :: Package common :: Module bundle
[hide private]

Source Code for Module flumotion.common.bundle

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_bundle -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  bundles of files used to implement caching over the network 
 24  """ 
 25   
 26  import errno 
 27  import md5 
 28  import os 
 29  import zipfile 
 30  import tempfile 
 31  import StringIO 
 32   
 33  from flumotion.common import errors, dag 
 34  from flumotion.common.python import makedirs 
 35   
 36  __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket'] 
 37  __version__ = "$Rev: 6982 $" 
 38   
 39   
40 -class BundledFile:
41 """ 42 I represent one file as managed by a bundler. 43 """
44 - def __init__(self, source, destination):
45 self.source = source 46 self.destination = destination 47 self._last_md5sum = None 48 self._last_timestamp = None 49 self.zipped = False
50
51 - def md5sum(self):
52 """ 53 Calculate the md5sum of the given file. 54 55 @returns: the md5 sum a 32 character string of hex characters. 56 """ 57 data = open(self.source, "r").read() 58 return md5.new(data).hexdigest()
59
60 - def timestamp(self):
61 """ 62 @returns: the last modified timestamp for the file. 63 """ 64 return os.path.getmtime(self.source)
65
66 - def hasChanged(self):
67 """ 68 Check if the file has changed since it was last checked. 69 70 @rtype: boolean 71 """ 72 73 # if it wasn't zipped yet, it needs zipping, so we pretend it 74 # was changed 75 # FIXME: move this out here 76 if not self.zipped: 77 return True 78 79 timestamp = self.timestamp() 80 # if file still has an old timestamp, it hasn't changed 81 # FIXME: looks bogus, shouldn't this check be != instead of <= ? 82 if self._last_timestamp and timestamp <= self._last_timestamp: 83 return False 84 self._last_timestamp = timestamp 85 86 # if the md5sum has changed, it has changed 87 md5sum = self.md5sum() 88 if self._last_md5sum != md5sum: 89 self._last_md5sum = md5sum 90 return True 91 92 return False
93
94 - def pack(self, zip):
95 self._last_timestamp = self.timestamp() 96 self._last_md5sum = self.md5sum() 97 zip.write(self.source, self.destination) 98 self.zipped = True
99
100 -class Bundle:
101 """ 102 I am a bundle of files, represented by a zip file and md5sum. 103 """
104 - def __init__(self, name):
105 self.zip = None 106 self.md5sum = None 107 self.name = name
108
109 - def setZip(self, zip):
110 """ 111 Set the bundle to the given data representation of the zip file. 112 """ 113 self.zip = zip 114 self.md5sum = md5.new(self.zip).hexdigest()
115
116 - def getZip(self):
117 """ 118 Get the bundle's zip data. 119 """ 120 return self.zip
121
122 -class Unbundler:
123 """ 124 I unbundle bundles by unpacking them in the given directory 125 under directories with the bundle's md5sum. 126 """
127 - def __init__(self, directory):
128 self._undir = directory
129
130 - def unbundlePathByInfo(self, name, md5sum):
131 """ 132 Return the full path where a bundle with the given name and md5sum 133 would be unbundled to. 134 """ 135 return os.path.join(self._undir, name, md5sum)
136
137 - def unbundlePath(self, bundle):
138 """ 139 Return the full path where this bundle will/would be unbundled to. 140 """ 141 return self.unbundlePathByInfo(bundle.name, bundle.md5sum)
142
143 - def unbundle(self, bundle):
144 """ 145 Unbundle the given bundle. 146 147 @type bundle: L{flumotion.common.bundle.Bundle} 148 149 @rtype: string 150 @returns: the full path to the directory where it was unpacked 151 """ 152 directory = self.unbundlePath(bundle) 153 154 filelike = StringIO.StringIO(bundle.getZip()) 155 zipFile = zipfile.ZipFile(filelike, "r") 156 zipFile.testzip() 157 158 filepaths = zipFile.namelist() 159 for filepath in filepaths: 160 path = os.path.join(directory, filepath) 161 parent = os.path.split(path)[0] 162 try: 163 makedirs(parent) 164 except OSError, err: 165 # Reraise error unless if it's an already existing 166 if err.errno != errno.EEXIST or not os.path.isdir(parent): 167 raise 168 data = zipFile.read(filepath) 169 170 # atomically write to path, see #373 171 fd, tempname = tempfile.mkstemp(dir=parent) 172 handle = os.fdopen(fd, 'wb') 173 handle.write(data) 174 handle.close() 175 176 # os.rename on Win32 is not deleting the target file 177 # if it exists, so remove it before 178 if os.path.exists(path): 179 os.unlink(path) 180 os.rename(tempname, path) 181 return directory
182
183 -class Bundler:
184 """ 185 I bundle files into a bundle so they can be cached remotely easily. 186 """
187 - def __init__(self, name):
188 """ 189 Create a new bundle. 190 """ 191 self._bundledFiles = {} # dictionary of BundledFile's indexed on path 192 self.name = name 193 self._bundle = Bundle(name)
194
195 - def add(self, source, destination = None):
196 """ 197 Add files to the bundle. 198 199 @param source: the path to the file to add to the bundle. 200 @param destination: a relative path to store this file in in the bundle. 201 If unspecified, this will be stored in the top level. 202 203 @returns: the path the file got stored as 204 """ 205 if destination == None: 206 destination = os.path.split(source)[1] 207 self._bundledFiles[source] = BundledFile(source, destination) 208 return destination
209
210 - def bundle(self):
211 """ 212 Bundle the files registered with the bundler. 213 214 @rtype: L{flumotion.common.bundle.Bundle} 215 """ 216 # rescan files registered in the bundle, and check if we need to 217 # rebuild the internal zip 218 if not self._bundle.getZip(): 219 self._bundle.setZip(self._buildzip()) 220 return self._bundle 221 222 update = False 223 for bundledFile in self._bundledFiles.values(): 224 if bundledFile.hasChanged(): 225 update = True 226 break 227 228 if update: 229 self._bundle.setZip(self._buildzip()) 230 231 return self._bundle
232 233 # build the zip file containing the files registered in the bundle 234 # and return the zip file data
235 - def _buildzip(self):
236 filelike = StringIO.StringIO() 237 zipFile = zipfile.ZipFile(filelike, "w") 238 for bundledFile in self._bundledFiles.values(): 239 bundledFile.pack(zipFile) 240 zipFile.close() 241 data = filelike.getvalue() 242 filelike.close() 243 return data
244
245 -class BundlerBasket:
246 """ 247 I manage bundlers that are registered through me. 248 """
249 - def __init__(self):
250 """ 251 Create a new bundler basket. 252 """ 253 self._bundlers = {} # bundler name -> bundle 254 255 self._files = {} # filename -> bundle name 256 self._imports = {} # import statements -> bundle name 257 258 self._graph = dag.DAG()
259
260 - def add(self, bundleName, source, destination=None):
261 """ 262 Add files to the bundler basket for the given bundle. 263 264 @param bundleName: the name of the bundle this file is a part of 265 @param source: the path to the file to add to the bundle 266 @param destination: a relative path to store this file in in the bundle. 267 If unspecified, this will be stored in the top level 268 """ 269 # get the bundler and create it if need be 270 if not bundleName in self._bundlers: 271 bundler = Bundler(bundleName) 272 self._bundlers[bundleName] = bundler 273 else: 274 bundler = self._bundlers[bundleName] 275 276 # add the file to the bundle and register 277 location = bundler.add(source, destination) 278 if location in self._files: 279 raise Exception("Cannot add %s to bundle %s, already in %s" % ( 280 location, bundleName, self._files[location])) 281 self._files[location] = bundleName 282 283 # add possible imports from this file 284 package = None 285 if location.endswith('.py'): 286 package = location[:-3] 287 elif location.endswith('.pyc'): 288 package = location[:-4] 289 290 if package: 291 if package.endswith('__init__'): 292 package = os.path.split(package)[0] 293 294 package = ".".join(package.split('/')) # win32 fixme 295 if package in self._imports: 296 raise Exception("Bundler %s already has import %s" % ( 297 bundleName, package)) 298 self._imports[package] = bundleName
299
300 - def depend(self, depender, *dependencies):
301 """ 302 Make the given bundle depend on the other given bundles. 303 304 @type depender: string 305 @type dependencies: list of strings 306 """ 307 # note that a bundler doesn't necessarily need to be registered yet 308 if not self._graph.hasNode(depender): 309 self._graph.addNode(depender) 310 for dep in dependencies: 311 if not self._graph.hasNode(dep): 312 self._graph.addNode(dep) 313 self._graph.addEdge(depender, dep)
314
315 - def getDependencies(self, bundlerName):
316 """ 317 Return names of all the dependencies of this bundle, including this 318 bundle itself. 319 The dependencies are returned in a correct depending order. 320 """ 321 if not bundlerName in self._bundlers: 322 raise errors.NoBundleError('Unknown bundle %s' % bundlerName) 323 elif not self._graph.hasNode(bundlerName): 324 return [bundlerName] 325 else: 326 return [bundlerName] + self._graph.getOffspring(bundlerName)
327
328 - def getBundlerByName(self, bundlerName):
329 """ 330 Return the bundle by name, or None if not found. 331 """ 332 if self._bundlers.has_key(bundlerName): 333 return self._bundlers[bundlerName] 334 return None
335
336 - def getBundlerNameByImport(self, importString):
337 """ 338 Return the bundler name by import statement, or None if not found. 339 """ 340 if self._imports.has_key(importString): 341 return self._imports[importString] 342 return None
343
344 - def getBundlerNameByFile(self, filename):
345 """ 346 Return the bundler name by filename, or None if not found. 347 """ 348 if self._files.has_key(filename): 349 return self._files[filename] 350 return None
351
352 - def getBundlerNames(self):
353 """ 354 Get all bundler names. 355 356 @rtype: list of str 357 @returns: a list of all bundler names in this basket. 358 """ 359 return self._bundlers.keys()
360
361 -class MergedBundler(Bundler):
362 """ 363 I am a bundler, with the extension that I can also bundle other 364 bundlers. 365 366 The effect is that when you call bundle() on a me, you get one 367 bundle with a union of all subbundlers' files, in addition to any 368 loose files that you added to me. 369 """
370 - def __init__(self, name='merged-bundle'):
371 Bundler.__init__(self, name) 372 self._subbundlers = {}
373
374 - def addBundler(self, bundler):
375 """Add to me all of the files managed by another bundler. 376 377 @param bundler: The bundler whose files you want in this 378 bundler. 379 @type bundler: L{Bundler} 380 """ 381 if bundler.name not in self._subbundlers: 382 self._subbundlers[bundler.name] = bundler 383 for bfile in bundler._files.values(): 384 self.add(bfile.source, bfile.destination)
385
386 - def getSubBundlers(self):
387 """ 388 @returns: A list of all of the bundlers that have been added to 389 me. 390 """ 391 return self._subbundlers.values()
392