flumotion.common.package

1 # -*- Mode: Python; test-case-name: flumotion.test.test_common_package -*- 2 # vi:si:et:sw=4:sts=4:ts=4 3 # 4 # Flumotion - a streaming media server 5 # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 6 # All rights reserved. 7 8 # This file may be distributed and/or modified under the terms of 9 # the GNU General Public License version 2 as published by 10 # the Free Software Foundation. 11 # This file is distributed without any warranty; without even the implied 12 # warranty of merchantability or fitness for a particular purpose. 13 # See "LICENSE.GPL" in the source distribution for more information. 14 15 # Licensees having purchased or holding a valid Flumotion Advanced 16 # Streaming Server license may use this file in accordance with the 17 # Flumotion Advanced Streaming Server Commercial License Agreement. 18 # See "LICENSE.Flumotion" in the source distribution for more information. 19 20 # Headers in this file shall remain intact. 21 22 """objects and functions used in dealing with packages 23 """ 24 25 import ihooks 26 import glob 27 import os 28 import sys 29 30 from twisted.python import rebuild, reflect 31 32 from flumotion.common import log, common 33 from flumotion.configure import configure 34 35 __version__ = "$Rev: 6982 $" 36 37

38 -class PackageHooks(ihooks.Hooks):

39 """ 40 I am an import Hooks object that makes sure that every package that gets 41 loaded has every necessary path in the module's __path__ list. 42 43 @type packager: L{Packager} 44 """ 45 packager = None 46

47 - def load_package(self, name, filename, file=None):

48 # this is only ever called the first time a package is imported 49 log.log('packager', 'load_package %s' % name) 50 ret = ihooks.Hooks.load_package(self, name, filename, file) 51 52 m = sys.modules[name] 53 54 packagePaths = self.packager.getPathsForPackage(name) 55 if not packagePaths: 56 return ret 57 58 # get full paths to the package 59 paths = [os.path.join(path, name.replace('.', os.sep)) for path in packagePaths] 60 for path in paths: 61 if not path in m.__path__: 62 log.log('packager', 'adding path %s for package %s' % ( 63 path, name)) 64 m.__path__.append(path) 65 66 return ret

67

68 -class Packager(log.Loggable):

69 """ 70 I am an object through which package paths can be registered, to support 71 the partitioning of the module import namespace across bundles. 72 """ 73 74 logCategory = 'packager' 75

76 - def __init__(self):

77 self._paths = {} # key -> package path registered with that key 78 self._packages = {} # package name -> keys for that package 79 self.install()

80

81 - def install(self):

82 """ 83 Install our custom importer that uses bundled packages. 84 """ 85 self.debug('installing custom importer') 86 self._hooks = PackageHooks() 87 self._hooks.packager = self 88 self._importer = ihooks.ModuleImporter() 89 self._importer.set_hooks(self._hooks) 90 self._importer.install()

91

92 - def getPathsForPackage(self, packageName):

93 """ 94 Return all absolute paths to the top level of a tree from which 95 (part of) the given package name can be imported. 96 """ 97 if packageName not in self._packages: 98 return None 99 100 return [self._paths[key] for key in self._packages[packageName]]

101

102 - def registerPackagePath(self, packagePath, key, prefix=configure.PACKAGE):

103 """ 104 Register a given path as a path that can be imported from. 105 Used to support partition of bundled code or import code from various 106 uninstalled location. 107 108 sys.path will also be changed to include this, and remove references 109 to older packagePath's for the same bundle. 110 111 @param packagePath: path to add under which the module namespaces live, 112 (ending in an md5sum, for flumotion purposes) 113 @type packagePath: string 114 @param key a unique id for the package being registered 115 @type key: string 116 @param prefix: prefix of the packages to be considered 117 @type prefix: string 118 """ 119 120 new = True 121 packagePath = os.path.abspath(packagePath) 122 if not os.path.exists(packagePath): 123 log.warning('bundle', 124 'registering a non-existing package path %s' % packagePath) 125 126 self.log('registering packagePath %s' % packagePath) 127 128 # check if a packagePath for this bundle was already registered 129 if key in self._paths: 130 oldPath = self._paths[key] 131 if packagePath == oldPath: 132 self.log('already registered %s for key %s' % ( 133 packagePath, key)) 134 return 135 new = False 136 137 # Find the packages in the path and sort them, 138 # the following algorithm only works if they're sorted. 139 # By sorting the list we can ensure that a parent package 140 # is always processed before one of its children 141 if not os.path.isdir(packagePath): 142 log.warning('bundle', 'package path not a dir: %s', 143 packagePath) 144 packageNames = [] 145 else: 146 packageNames = _findPackageCandidates(packagePath, prefix) 147 148 if not packageNames: 149 log.log('bundle', 150 'packagePath %s does not have candidates starting with %s' % 151 (packagePath, prefix)) 152 return 153 packageNames.sort() 154 155 self.log('package candidates %r' % packageNames) 156 157 if not new: 158 # it already existed, and now it's a different path 159 log.log('bundle', 160 'replacing old path %s with new path %s for key %s' % ( 161 oldPath, packagePath, key)) 162 163 if oldPath in sys.path: 164 log.log('bundle', 165 'removing old packagePath %s from sys.path' % oldPath) 166 sys.path.remove(oldPath) 167 168 # clear this key from our name -> key cache 169 for keys in self._packages.values(): 170 if key in keys: 171 keys.remove(key) 172 173 self._paths[key] = packagePath 174 175 # put packagePath at the top of sys.path if not in there 176 if not packagePath in sys.path: 177 self.log('adding packagePath %s to sys.path' % packagePath) 178 sys.path.insert(0, packagePath) 179 180 # update our name->keys cache 181 for name in packageNames: 182 if name not in self._packages: 183 self._packages[name] = [key] 184 else: 185 self._packages[name].insert(0, key) 186 187 self.log('packagePath %s has packageNames %r' % ( 188 packagePath, packageNames)) 189 # since we want sub-modules to be fixed up before parent packages, 190 # we reverse the list 191 packageNames.reverse() 192 193 for packageName in packageNames: 194 if packageName not in sys.modules: 195 continue 196 self.log('fixing up %s ...' % packageName) 197 198 # the package is imported, so mess with __path__ and rebuild 199 package = sys.modules.get(packageName) 200 for path in package.__path__: 201 if not new and path.startswith(oldPath): 202 self.log('%s.__path__ before remove %r' % ( 203 packageName, package.__path__)) 204 self.log('removing old %s from %s.__path__' % ( 205 path, name)) 206 package.__path__.remove(path) 207 self.log('%s.__path__ after remove %r' % ( 208 packageName, package.__path__)) 209 210 # move the new path to the top 211 # insert at front because FLU_REGISTRY_PATH paths should override 212 # base components, and because subsequent reload() should prefer 213 # the latest registered path 214 newPath = os.path.join(packagePath, 215 packageName.replace('.', os.sep)) 216 217 # if path already at position 0, everything's fine 218 # if it's in there at another place, it needs to move to front 219 # if not in there, it needs to be put in front 220 if len(package.__path__) == 0: 221 # FIXME: this seems to happen to e.g. flumotion.component.base 222 # even when it was just rebuilt and had the __path__ set 223 # can be triggered by choosing a admin_gtk depending on 224 # the base admin_gtk where the base admin_gtk changes 225 self.debug('WARN: package %s does not have __path__ values' % ( 226 packageName)) 227 elif package.__path__[0] == newPath: 228 self.log('path %s already at start of %s.__path__' % ( 229 newPath, packageName)) 230 continue 231 232 if newPath in package.__path__: 233 package.__path__.remove(newPath) 234 self.log('moving %s to front of %s.__path__' % ( 235 newPath, packageName)) 236 else: 237 self.log('inserting new %s into %s.__path__' % ( 238 newPath, packageName)) 239 package.__path__.insert(0, newPath) 240 241 # Rebuilding these packages just to get __path__ fixed in 242 # seems not necessary - but re-enable it if it breaks 243 # self.log('rebuilding package %s from paths %r' % (packageName, 244 # package.__path__)) 245 # rebuild.rebuild(package) 246 # self.log('rebuilt package %s with paths %r' % (packageName, 247 # package.__path__)) 248 self.log('fixed up %s, __path__ %s ...' % (packageName, package.__path__)) 249 250 # now rebuild all non-package modules in this packagePath if this 251 # is not a new package 252 if not new: 253 self.log('finding end module candidates') 254 if not os.path.isdir(packagePath): 255 log.warning('bundle', 'package path not a dir: %s', 256 path) 257 moduleNames = [] 258 else: 259 moduleNames = findEndModuleCandidates(packagePath, prefix) 260 self.log('end module candidates to rebuild: %r' % moduleNames) 261 for name in moduleNames: 262 if name in sys.modules: 263 # fixme: isn't sys.modules[name] sufficient? 264 self.log("rebuilding non-package module %s" % name) 265 try: 266 module = reflect.namedAny(name) 267 except AttributeError: 268 log.warning('bundle', 269 "could not reflect non-package module %s" % name) 270 continue 271 272 if hasattr(module, '__path__'): 273 self.log('rebuilding module %s with paths %r' % (name, 274 module.__path__)) 275 rebuild.rebuild(module) 276 #if paths: 277 # module.__path__ = paths 278 279 self.log('registered packagePath %s for key %s' % (packagePath, key))

280

281 - def unregister(self):

282 """ 283 Unregister all previously registered package paths, and uninstall 284 the custom importer. 285 """ 286 for path in self._paths.values(): 287 if path in sys.path: 288 self.log('removing packagePath %s from sys.path' % path) 289 sys.path.remove(path) 290 self._paths = {} 291 self._packages = {} 292 self.debug('uninstalling custom importer') 293 self._importer.uninstall()

294

295 -def _listDirRecursively(path):

296 """ 297 I'm similar to os.listdir, but I work recursively and only return 298 directories containing python code. 299 300 @param path: the path 301 @type path: string 302 """ 303 retval = [] 304 try: 305 files = os.listdir(path) 306 except OSError: 307 pass 308 else: 309 for f in files: 310 # this only adds directories since files are not returned 311 p = os.path.join(path, f) 312 if os.path.isdir(p) and f != '.svn': 313 retval += _listDirRecursively(p) 314 315 if glob.glob(os.path.join(path, '*.py*')): 316 retval.append(path) 317 318 return retval

319

320 -def _listPyFileRecursively(path):

321 """ 322 I'm similar to os.listdir, but I work recursively and only return 323 files representing python non-package modules. 324 325 @param path: the path 326 @type path: string 327 328 @rtype: list 329 @returns: list of files underneath the given path containing python code 330 """ 331 retval = [] 332 333 # get all the dirs containing python code 334 dirs = _listDirRecursively(path) 335 336 for directory in dirs: 337 pyfiles = glob.glob(os.path.join(directory, '*.py*')) 338 dontkeep = glob.glob(os.path.join(directory, '*__init__.py*')) 339 for f in dontkeep: 340 if f in pyfiles: 341 pyfiles.remove(f) 342 343 retval.extend(pyfiles) 344 345 return retval

346

347 -def _findPackageCandidates(path, prefix=configure.PACKAGE):

348 """ 349 I take a directory and return a list of candidate python packages 350 under that directory that start with the given prefix. 351 A package is a module containing modules; typically the directory 352 with the same name as the package contains __init__.py 353 354 @param path: the path 355 @type path: string 356 """ 357 # this function also "guesses" candidate packages when __init__ is missing 358 # so a bundle with only a subpackage is also detected 359 dirs = _listDirRecursively(os.path.join(path, prefix)) 360 361 # chop off the base path to get a list of "relative" bundlespace paths 362 bundlePaths = [x[len(path) + 1:] for x in dirs] 363 364 # remove some common candidates, like .svn subdirs, or containing - 365 bundlePaths = [path for path in bundlePaths if path.find('.svn') == -1] 366 bundlePaths = [path for path in bundlePaths if path.find('-') == -1] 367 368 # convert paths to module namespace 369 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths] 370 371 # now make sure that all parent packages for each package are listed 372 # as well 373 packages = {} 374 for name in bundlePackages: 375 packages[name] = 1 376 parts = name.split(".") 377 build = None 378 for p in parts: 379 if not build: 380 build = p 381 else: 382 build = build + "." + p 383 packages[build] = 1 384 385 bundlePackages = packages.keys() 386 387 # sort them so that depending packages are after higher-up packages 388 bundlePackages.sort() 389 390 return bundlePackages

391

392 -def findEndModuleCandidates(path, prefix=configure.PACKAGE):

393 """ 394 I take a directory and return a list of candidate python end modules 395 (i.e., non-package modules) for the given module prefix. 396 397 @param path: the path under which to search for end modules 398 @type path: string 399 @param prefix: module prefix to check candidates under 400 @type prefix: string 401 """ 402 pathPrefix = "/".join(prefix.split(".")) 403 files = _listPyFileRecursively(os.path.join(path, pathPrefix)) 404 405 # chop off the base path to get a list of "relative" import space paths 406 importPaths = [x[len(path) + 1:] for x in files] 407 408 # remove some common candidates, like .svn subdirs, or containing - 409 importPaths = [path for path in importPaths if path.find('.svn') == -1] 410 importPaths = [path for path in importPaths if path.find('-') == -1] 411 412 # convert paths to module namespace 413 endModules = [common.pathToModuleName(x) for x in importPaths] 414 415 # remove all not starting with prefix 416 endModules = [module for module in endModules 417 if module and module.startswith(prefix)] 418 419 # sort them so that depending packages are after higher-up packages 420 endModules.sort() 421 422 # make unique 423 res = {} 424 for b in endModules: res[b] = 1 425 426 return res.keys()

427 428 # singleton factory function 429 __packager = None 430

431 -def getPackager():

432 """ 433 Return the (unique) packager. 434 435 @rtype: L{Packager} 436 """ 437 global __packager 438 if not __packager: 439 __packager = Packager() 440 441 return __packager

442

Source Code for Module flumotion.common.package