Package flumotion :: Package common :: Module bundle
[hide private]

Source Code for Module flumotion.common.bundle

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_bundle -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  bundles of files used to implement caching over the network 
 24  """ 
 25   
 26  import errno 
 27  import md5 
 28  import os 
 29  import sys 
 30  import zipfile 
 31  import StringIO 
 32   
 33  from flumotion.common import errors, dag 
 34   
 35  __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket'] 
 36   
37 -class BundledFile:
38 """ 39 I represent one file as managed by a bundler. 40 """
41 - def __init__(self, source, destination):
42 self.source = source 43 self.destination = destination 44 self._last_md5sum = self.md5sum() 45 self._last_timestamp = self.timestamp() 46 self.zipped = False
47
48 - def md5sum(self):
49 """ 50 Calculate the md5sum of the given file. 51 52 @returns: the md5 sum a 32 character string of hex characters. 53 """ 54 data = open(self.source, "r").read() 55 return md5.new(data).hexdigest()
56
57 - def timestamp(self):
58 """ 59 @returns: the last modified timestamp for the file. 60 """ 61 return os.path.getmtime(self.source)
62
63 - def hasChanged(self):
64 """ 65 Check if the file has changed since it was last checked. 66 67 @rtype: boolean 68 """ 69 70 # if it wasn't zipped yet, it needs zipping, so we pretend it 71 # was changed 72 # FIXME: move this out here 73 if not self.zipped: 74 return True 75 76 timestamp = self.timestamp() 77 # if file still has an old timestamp, it hasn't changed 78 if timestamp <= self._last_timestamp: 79 return False 80 self._last_timestamp = timestamp 81 82 # if the md5sum has changed, it has changed 83 md5sum = self.md5sum() 84 if self._last_md5sum != md5sum: 85 self._last_md5sum = md5sum 86 return True 87 88 return False
89
90 -class Bundle:
91 """ 92 I am a bundle of files, represented by a zip file and md5sum. 93 """
94 - def __init__(self, name):
95 self.zip = None 96 self.md5sum = None 97 self.name = name
98
99 - def setZip(self, zip):
100 """ 101 Set the bundle to the given data representation of the zip file. 102 """ 103 self.zip = zip 104 self.md5sum = md5.new(self.zip).hexdigest()
105
106 - def getZip(self):
107 """ 108 Get the bundle's zip data. 109 """ 110 return self.zip
111
112 -class Unbundler:
113 """ 114 I unbundle bundles by unpacking them in the given directory 115 under directories with the bundle's md5sum. 116 """
117 - def __init__(self, directory):
118 self._undir = directory
119
120 - def unbundlePathByInfo(self, name, md5sum):
121 """ 122 Return the full path where a bundle with the given name and md5sum 123 would be unbundled to. 124 """ 125 return os.path.join(self._undir, name, md5sum)
126
127 - def unbundlePath(self, bundle):
128 """ 129 Return the full path where this bundle will/would be unbundled to. 130 """ 131 return self.unbundlePathByInfo(bundle.name, bundle.md5sum)
132
133 - def unbundle(self, bundle):
134 """ 135 Unbundle the given bundle. 136 137 @type bundle: L{flumotion.common.bundle.Bundle} 138 139 @rtype: string 140 @returns: the full path to the directory where it was unpacked 141 """ 142 dir = self.unbundlePath(bundle) 143 144 filelike = StringIO.StringIO(bundle.getZip()) 145 zip = zipfile.ZipFile(filelike, "r") 146 zip.testzip() 147 148 filepaths = zip.namelist() 149 for filepath in filepaths: 150 path = os.path.join(dir, filepath) 151 parent = os.path.split(path)[0] 152 try: 153 os.makedirs(parent) 154 except OSError, err: 155 # Reraise error unless if it's an already existing 156 if err.errno != errno.EEXIST or not os.path.isdir(parent): 157 raise 158 data = zip.read(filepath) 159 handle = open(path, 'wb') 160 handle.write(data) 161 handle.close() 162 return dir
163
164 -class Bundler:
165 """ 166 I bundle files into a bundle so they can be cached remotely easily. 167 """
168 - def __init__(self, name):
169 """ 170 Create a new bundle. 171 """ 172 self._files = {} # dictionary of BundledFile's indexed on path 173 self.name = name 174 self._bundle = Bundle(name)
175
176 - def add(self, source, destination = None):
177 """ 178 Add files to the bundle. 179 180 @param source: the path to the file to add to the bundle. 181 @param destination: a relative path to store this file in in the bundle. 182 If unspecified, this will be stored in the top level. 183 184 @returns: the path the file got stored as 185 """ 186 if destination == None: 187 destination = os.path.split(source)[1] 188 self._files[source] = BundledFile(source, destination) 189 return destination
190
191 - def bundle(self):
192 """ 193 Bundle the files registered with the bundler. 194 195 @rtype: L{flumotion.common.bundle.Bundle} 196 """ 197 # rescan files registered in the bundle, and check if we need to 198 # rebuild the internal zip 199 if not self._bundle.getZip(): 200 self._bundle.setZip(self._buildzip()) 201 return self._bundle 202 203 update = False 204 for file in self._files.values(): 205 if file.hasChanged(): 206 update = True 207 208 if update: 209 self._bundle.setZip(self._buildzip()) 210 211 return self._bundle
212 213 # build the zip file containing the files registered in the bundle 214 # and return the zip file data
215 - def _buildzip(self):
216 filelike = StringIO.StringIO() 217 zip = zipfile.ZipFile(filelike, "w") 218 for path in self._files.keys(): 219 bf = self._files[path] 220 self._files[path].zipped = True 221 zip.write(bf.source, bf.destination) 222 zip.close() 223 data = filelike.getvalue() 224 filelike.close() 225 return data
226
227 -class BundlerBasket:
228 """ 229 I manage bundlers that are registered through me. 230 """
231 - def __init__(self):
232 """ 233 Create a new bundler basket. 234 """ 235 self._bundlers = {} # bundler name -> bundle 236 237 self._files = {} # filename -> bundle name 238 self._imports = {} # import statements -> bundle name 239 240 self._graph = dag.DAG()
241
242 - def add(self, bundleName, source, destination = None):
243 """ 244 Add files to the bundler basket for the given bundle. 245 246 @param bundleName: the name of the bundle this file is a part of 247 @param source: the path to the file to add to the bundle 248 @param destination: a relative path to store this file in in the bundle. 249 If unspecified, this will be stored in the top level 250 """ 251 # get the bundler and create it if need be 252 if not bundleName in self._bundlers.keys(): 253 bundler = Bundler(bundleName) 254 self._bundlers[bundleName] = bundler 255 else: 256 bundler = self._bundlers[bundleName] 257 258 # add the file to the bundle and register 259 location = bundler.add(source, destination) 260 if self._files.has_key(location): 261 raise Exception("Cannot add %s to bundle %s, already in %s" % ( 262 location, bundleName, self._files[location])) 263 self._files[location] = bundleName 264 265 # add possible imports from this file 266 package = None 267 if location.endswith('.py'): 268 package = location[:-3] 269 elif location.endswith('.pyc'): 270 package = location[:-4] 271 272 if package: 273 if package.endswith('__init__'): 274 package = os.path.split(package)[0] 275 276 package = ".".join(package.split('/')) # win32 fixme 277 if self._imports.has_key(package): 278 raise Exception("Bundler %s already has import %s" % ( 279 bundleName, package)) 280 self._imports[package] = bundleName
281
282 - def depend(self, depender, *dependencies):
283 """ 284 Make the given bundle depend on the other given bundles. 285 286 @type depender: string 287 @type dependencies: list of strings 288 """ 289 # note that a bundler doesn't necessarily need to be registered yet 290 if not self._graph.hasNode(depender): 291 self._graph.addNode(depender) 292 for dep in dependencies: 293 if not self._graph.hasNode(dep): 294 self._graph.addNode(dep) 295 self._graph.addEdge(depender, dep)
296
297 - def getDependencies(self, bundlerName):
298 """ 299 Return names of all the dependencies of this bundle, including this 300 bundle itself. 301 The dependencies are returned in a correct depending order. 302 """ 303 if not bundlerName in self._bundlers: 304 raise errors.NoBundleError('Unknown bundle %s' % bundlerName) 305 return [bundlerName,] + self._graph.getOffspring(bundlerName)
306
307 - def getBundlerByName(self, bundlerName):
308 """ 309 Return the bundle by name, or None if not found. 310 """ 311 if self._bundlers.has_key(bundlerName): 312 return self._bundlers[bundlerName] 313 return None
314
315 - def getBundlerNameByImport(self, importString):
316 """ 317 Return the bundler name by import statement, or None if not found. 318 """ 319 if self._imports.has_key(importString): 320 return self._imports[importString] 321 return None
322
323 - def getBundlerNameByFile(self, filename):
324 """ 325 Return the bundler name by filename, or None if not found. 326 """ 327 if self._files.has_key(filename): 328 return self._files[filename] 329 return None
330
331 -class MergedBundler(Bundler):
332 """ 333 I am a bundler, with the extension that I can also bundle other 334 bundlers. 335 336 The effect is that when you call bundle() on a me, you get one 337 bundle with a union of all subbundlers' files, in addition to any 338 loose files that you added to me. 339 """
340 - def __init__(self, name='merged-bundle'):
341 Bundler.__init__(self, name) 342 self._subbundlers = {}
343
344 - def addBundler(self, bundler):
345 """Add to me all of the files managed by another bundler. 346 347 @param bundler: The bundler whose files you want in this 348 bundler. 349 @type bundler: L{Bundler} 350 """ 351 if bundler.name not in self._subbundlers: 352 self._subbundlers[bundler.name] = bundler 353 for bfile in bundler._files.values(): 354 self.add(bfile.source, bfile.destination)
355
356 - def getSubBundlers(self):
357 """ 358 @returns: A list of all of the bundlers that have been added to 359 me. 360 """ 361 return self._subbundlers.values()
362
363 -def makeBundleFromLoadedModules(outfile, outreg, *prefixes):
364 """ 365 Make a bundle from a subset of all loaded modules, also writing out 366 a registry file that can apply to that subset of the global 367 registry. Suitable for use as a FLU_ATEXIT handler. 368 369 @param outfile: The path to which a zip file will be written. 370 @type outfile: str 371 @param outreg: The path to which a registry file will be written. 372 @type outreg: str 373 @param prefixes: A list of prefixes to which to limit the export. If 374 not given, package up all modules. For example, "flumotion" would 375 limit the output to modules that start with "flumotion". 376 @type prefixes: list of str 377 """ 378 from flumotion.common import registry, log 379 from twisted.python import reflect 380 381 def getUsedModules(prefixes): 382 ret = {} 383 for modname in sys.modules: 384 if prefixes and not filter(modname.startswith, prefixes): 385 continue 386 try: 387 module = reflect.namedModule(modname) 388 if hasattr(module, '__file__'): 389 ret[modname] = module 390 else: 391 log.info('makebundle', 'Module %s has no file', module) 392 except ImportError: 393 log.info('makebundle', 'Could not import %s', modname) 394 return ret
395 396 def calculateModuleBundleMap(): 397 allbundles = registry.getRegistry().getBundles() 398 ret = {} 399 for bundle in allbundles: 400 for directory in bundle.getDirectories(): 401 for file in directory.getFiles(): 402 path = os.path.join(directory.getName(), file.getLocation()) 403 parts = path.split(os.path.sep) 404 if parts[-1].startswith('__init__.py'): 405 parts.pop() 406 elif parts[-1].endswith('.py'): 407 parts[-1] = parts[-1][:-3] 408 else: 409 # not a bundled module 410 continue 411 modname = '.'.join(parts) 412 ret[modname] = bundle 413 return ret 414 415 def makeMergedBundler(modules, modulebundlemap): 416 ret = MergedBundler() 417 basket = registry.getRegistry().makeBundlerBasket() 418 for modname in modules: 419 modfilename = modules[modname].__file__ 420 if modname in modulebundlemap: 421 bundleName = modulebundlemap[modname].getName() 422 for depBundleName in basket.getDependencies(bundleName): 423 ret.addBundler(basket.getBundlerByName(depBundleName)) 424 else: 425 if modfilename.endswith('.pyc'): 426 modfilename = modfilename[:-1] 427 if os.path.isdir(modfilename): 428 with_init = os.path.join(modfilename, '__init__.py') 429 if os.path.exists(with_init): 430 modfilename = with_init 431 nparts = len(modname.split('.')) 432 if '__init__' in modfilename: 433 nparts += 1 434 relpath = os.path.join(*modfilename.split(os.path.sep)[-nparts:]) 435 ret.add(modfilename, relpath) 436 return ret 437 438 modules = getUsedModules(prefixes) 439 modulebundlemap = calculateModuleBundleMap() 440 bundler = makeMergedBundler(modules, modulebundlemap) 441 442 print 'Writing bundle to', outfile 443 open(outfile, 'w').write(bundler.bundle().getZip()) 444 445 print 'Writing registry to', outreg 446 bundlers_used = [b.name for b in bundler.getSubBundlers()] 447 regwriter = registry.RegistrySubsetWriter(onlyBundles=bundlers_used) 448 regwriter.dump(open(outreg, 'w')) 449