Package flumotion :: Package common :: Module package
[hide private]

Source Code for Module flumotion.common.package

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common_package -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  objects and functions used in dealing with packages 
 24  """ 
 25   
 26  import ihooks 
 27  import os 
 28  import sys 
 29  import glob 
 30   
 31  from flumotion.common import log, common 
 32  from twisted.python import rebuild, reflect 
 33   
34 -class PackageHooks(ihooks.Hooks):
35 """ 36 I am an import Hooks object that makes sure that every package that gets 37 loaded has every necessary path in the module's __path__ list. 38 39 @type packager: L{Packager} 40 """ 41 packager = None 42
43 - def load_package(self, name, filename, file=None):
44 # this is only ever called the first time a package is imported 45 log.log('packager', 'load_package %s' % name) 46 ret = ihooks.Hooks.load_package(self, name, filename, file) 47 48 m = sys.modules[name] 49 50 packagePaths = self.packager.getPathsForPackage(name) 51 if not packagePaths: 52 return ret 53 54 # get full paths to the package 55 paths = [os.path.join(path, name.replace('.', os.sep)) for path in packagePaths] 56 for path in paths: 57 if not path in m.__path__: 58 log.log('packager', 'adding path %s for package %s' % ( 59 path, name)) 60 m.__path__.append(path) 61 62 return ret
63
64 -class Packager(log.Loggable):
65 """ 66 I am an object through which package paths can be registered, to support 67 the partitioning of the module import namespace across bundles. 68 """ 69 70 logCategory = 'packager' 71
72 - def __init__(self):
73 self._paths = {} # key -> package path registered with that key 74 self._packages = {} # package name -> keys for that package 75 self.install()
76
77 - def install(self):
78 """ 79 Install our custom importer that uses bundled packages. 80 """ 81 self.debug('installing custom importer') 82 self._hooks = PackageHooks() 83 self._hooks.packager = self 84 self._importer = ihooks.ModuleImporter() 85 self._importer.set_hooks(self._hooks) 86 self._importer.install()
87
88 - def getPathsForPackage(self, packageName):
89 """ 90 Return all absolute paths to the top level of a tree from which 91 (part of) the given package name can be imported. 92 """ 93 if not packageName in self._packages.keys(): 94 return None 95 96 return [self._paths[key] for key in self._packages[packageName]]
97
98 - def registerPackagePath(self, packagePath, key, prefix='flumotion'):
99 """ 100 Register a given path as a path that can be imported from. 101 Used to support partition of bundled code or import code from various 102 uninstalled location. 103 104 sys.path will also be changed to include this, and remove references 105 to older packagePath's for the same bundle. 106 107 @param packagePath: path to add under which the module namespaces live, 108 (ending in an md5sum, for flumotion purposes) 109 @type packagePath: string 110 @param key a unique id for the package being registered 111 @type key: string 112 @param prefix: prefix of the packages to be considered 113 @type prefix: string 114 """ 115 116 new = True 117 packagePath = os.path.abspath(packagePath) 118 if not os.path.exists(packagePath): 119 log.warning('bundle', 120 'registering a non-existing package path %s' % packagePath) 121 122 self.log('registering packagePath %s' % packagePath) 123 124 # check if a packagePath for this bundle was already registered 125 if key in self._paths.keys(): 126 oldPath = self._paths[key] 127 if packagePath == oldPath: 128 self.log('already registered %s for key %s' % ( 129 packagePath, key)) 130 return 131 new = False 132 133 # Find the packages in the path and sort them, 134 # the following algorithm only works if they're sorted. 135 # By sorting the list we can ensure that a parent package 136 # is always processed before one of its children 137 packageNames = _findPackageCandidates(packagePath, prefix) 138 139 if not packageNames: 140 log.log('bundle', 141 'packagePath %s does not have candidates starting with %s' % 142 (packagePath, prefix)) 143 return 144 packageNames.sort() 145 146 self.log('package candidates %r' % packageNames) 147 148 if not new: 149 # it already existed, and now it's a different path 150 log.log('bundle', 151 'replacing old path %s with new path %s for key %s' % ( 152 oldPath, packagePath, key)) 153 154 if oldPath in sys.path: 155 log.log('bundle', 156 'removing old packagePath %s from sys.path' % oldPath) 157 sys.path.remove(oldPath) 158 159 # clear this key from our name -> key cache 160 for keys in self._packages.values(): 161 if key in keys: 162 keys.remove(key) 163 164 self._paths[key] = packagePath 165 166 # put packagePath at the top of sys.path if not in there 167 if not packagePath in sys.path: 168 self.log('adding packagePath %s to sys.path' % packagePath) 169 sys.path.insert(0, packagePath) 170 171 # update our name->keys cache 172 for name in packageNames: 173 if not name in self._packages.keys(): 174 self._packages[name] = [key] 175 else: 176 self._packages[name].insert(0, key) 177 178 self.log('packagePath %s has packageNames %r' % ( 179 packagePath, packageNames)) 180 # since we want sub-modules to be fixed up before parent packages, 181 # we reverse the list 182 packageNames.reverse() 183 184 for packageName in packageNames: 185 if packageName not in sys.modules.keys(): 186 continue 187 self.log('fixing up %s ...' % packageName) 188 189 # the package is imported, so mess with __path__ and rebuild 190 package = sys.modules.get(packageName) 191 for path in package.__path__: 192 if not new and path.startswith(oldPath): 193 self.log('%s.__path__ before remove %r' % ( 194 packageName, package.__path__)) 195 self.log('removing old %s from %s.__path__' % ( 196 path, name)) 197 package.__path__.remove(path) 198 self.log('%s.__path__ after remove %r' % ( 199 packageName, package.__path__)) 200 201 # move the new path to the top 202 # insert at front because FLU_REGISTRY_PATH paths should override 203 # base components, and because subsequent reload() should prefer 204 # the latest registered path 205 newPath = os.path.join(packagePath, 206 packageName.replace('.', os.sep)) 207 208 # if path already at position 0, everything's fine 209 # if it's in there at another place, it needs to move to front 210 # if not in there, it needs to be put in front 211 if len(package.__path__) == 0: 212 # FIXME: this seems to happen to e.g. flumotion.component.base 213 # even when it was just rebuilt and had the __path__ set 214 # can be triggered by choosing a admin_gtk depending on 215 # the base admin_gtk where the base admin_gtk changes 216 self.debug('WARN: package %s does not have __path__ values' % ( 217 packageName)) 218 elif package.__path__[0] == newPath: 219 self.log('path %s already at start of %s.__path__' % ( 220 newPath, packageName)) 221 continue 222 223 if newPath in package.__path__: 224 package.__path__.remove(newPath) 225 self.log('moving %s to front of %s.__path__' % ( 226 newPath, packageName)) 227 else: 228 self.log('inserting new %s into %s.__path__' % ( 229 newPath, packageName)) 230 package.__path__.insert(0, newPath) 231 232 # Rebuilding these packages just to get __path__ fixed in 233 # seems not necessary - but re-enable it if it breaks 234 # self.log('rebuilding package %s from paths %r' % (packageName, 235 # package.__path__)) 236 # rebuild.rebuild(package) 237 # self.log('rebuilt package %s with paths %r' % (packageName, 238 # package.__path__)) 239 self.log('fixed up %s, __path__ %s ...' % (packageName, package.__path__)) 240 241 # now rebuild all non-package modules in this packagePath if this 242 # is not a new package 243 if not new: 244 self.log('finding end module candidates') 245 moduleNames = findEndModuleCandidates(packagePath, prefix) 246 self.log('end module candidates to rebuild: %r' % moduleNames) 247 for name in moduleNames: 248 if name in sys.modules: 249 # fixme: isn't sys.modules[name] sufficient? 250 self.log("rebuilding non-package module %s" % name) 251 try: 252 module = reflect.namedAny(name) 253 except AttributeError: 254 log.warning('bundle', 255 "could not reflect non-package module %s" % name) 256 continue 257 258 if hasattr(module, '__path__'): 259 self.log('rebuilding module %s with paths %r' % (name, 260 module.__path__)) 261 rebuild.rebuild(module) 262 #if paths: 263 # module.__path__ = paths 264 265 self.log('registered packagePath %s for key %s' % (packagePath, key))
266
267 - def unregister(self):
268 """ 269 Unregister all previously registered package paths, and uninstall 270 the custom importer. 271 """ 272 for path in self._paths.values(): 273 if path in sys.path: 274 self.log('removing packagePath %s from sys.path' % path) 275 sys.path.remove(path) 276 self._paths = {} 277 self._packages = {} 278 self.debug('uninstalling custom importer') 279 self._importer.uninstall()
280
281 -def _listDirRecursively(path):
282 """ 283 I'm similar to os.listdir, but I work recursively and only return 284 directories containing python code. 285 286 @param path: the path 287 @type path: string 288 """ 289 retval = [] 290 # files are never returned, only directories 291 if not os.path.isdir(path): 292 return retval 293 294 try: 295 files = os.listdir(path) 296 except OSError: 297 pass 298 else: 299 for f in files: 300 # this only adds directories since files are not returned 301 retval += _listDirRecursively(os.path.join(path, f)) 302 303 if glob.glob(os.path.join(path, '*.py*')): 304 retval.append(path) 305 306 return retval
307
308 -def _listPyFileRecursively(path):
309 """ 310 I'm similar to os.listdir, but I work recursively and only return 311 files representing python non-package modules. 312 313 @param path: the path 314 @type path: string 315 316 @rtype: list 317 @returns: list of files underneath the given path containing python code 318 """ 319 retval = [] 320 321 # get all the dirs containing python code 322 dirs = _listDirRecursively(path) 323 324 for dir in dirs: 325 pyfiles = glob.glob(os.path.join(dir, '*.py*')) 326 dontkeep = glob.glob(os.path.join(dir, '*__init__.py*')) 327 for f in dontkeep: 328 if f in pyfiles: 329 pyfiles.remove(f) 330 331 retval.extend(pyfiles) 332 333 return retval
334
335 -def _findPackageCandidates(path, prefix='flumotion'):
336 """ 337 I take a directory and return a list of candidate python packages 338 under that directory that start with the given prefix. 339 A package is a module containing modules; typically the directory 340 with the same name as the package contains __init__.py 341 342 @param path: the path 343 @type path: string 344 """ 345 # this function also "guesses" candidate packages when __init__ is missing 346 # so a bundle with only a subpackage is also detected 347 dirs = _listDirRecursively(os.path.join(path, prefix)) 348 349 # chop off the base path to get a list of "relative" bundlespace paths 350 bundlePaths = [x[len(path) + 1:] for x in dirs] 351 352 # remove some common candidates, like .svn subdirs, or containing - 353 isNotSvn = lambda x: x.find('.svn') == -1 354 bundlePaths = filter(isNotSvn, bundlePaths) 355 isNotDashed = lambda x: x.find('-') == -1 356 bundlePaths = filter(isNotDashed, bundlePaths) 357 358 # convert paths to module namespace 359 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths] 360 361 # now make sure that all parent packages for each package are listed 362 # as well 363 packages = {} 364 for name in bundlePackages: 365 packages[name] = 1 366 parts = name.split(".") 367 build = None 368 for p in parts: 369 if not build: 370 build = p 371 else: 372 build = build + "." + p 373 packages[build] = 1 374 375 bundlePackages = packages.keys() 376 377 # sort them so that depending packages are after higher-up packages 378 bundlePackages.sort() 379 380 return bundlePackages
381
382 -def findEndModuleCandidates(path, prefix='flumotion'):
383 """ 384 I take a directory and return a list of candidate python end modules 385 (i.e., non-package modules) for the given module prefix. 386 387 @param path: the path under which to search for end modules 388 @type path: string 389 @param prefix: module prefix to check candidates under 390 @type prefix: string 391 """ 392 pathPrefix = "/".join(prefix.split(".")) 393 files = _listPyFileRecursively(os.path.join(path, pathPrefix)) 394 395 # chop off the base path to get a list of "relative" import space paths 396 importPaths = [x[len(path) + 1:] for x in files] 397 398 # remove some common candidates, like .svn subdirs, or containing - 399 isNotSvn = lambda x: x.find('.svn') == -1 400 importPaths = filter(isNotSvn, importPaths) 401 isNotDashed = lambda x: x.find('-') == -1 402 importPaths = filter(isNotDashed, importPaths) 403 404 # convert paths to module namespace 405 endModules = [common.pathToModuleName(x) for x in importPaths] 406 407 # remove all not starting with prefix 408 isInPrefix = lambda x: x and x.startswith(prefix) 409 endModules = filter(isInPrefix, endModules) 410 411 # sort them so that depending packages are after higher-up packages 412 endModules.sort() 413 414 # make unique 415 res = {} 416 for b in endModules: res[b] = 1 417 418 return res.keys()
419 420 # singleton factory function 421 __packager = None 422
423 -def getPackager():
424 """ 425 Return the (unique) packager. 426 427 @rtype: L{Packager} 428 """ 429 global __packager 430 if not __packager: 431 __packager = Packager() 432 433 return __packager
434