Package flumotion :: Package common :: Module common
[hide private]

Source Code for Module flumotion.common.common

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_common -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  small common functions used by all processes 
 24  """ 
 25   
 26  import errno 
 27  import os  
 28  import sys 
 29  import time 
 30  import signal 
 31   
 32  from twisted.python import reflect, rebuild, components 
 33  from twisted.internet import address 
 34  import twisted.copyright 
 35   
 36  from flumotion.common import log 
 37   
 38  # Note: This module is loaded very early on, so 
 39  #       don't add any extra flumotion imports unless you 
 40  #       really know what you're doing. 
 41  from flumotion.configure import configure 
 42   
43 -def formatStorage(units, precision = 2):
44 """ 45 Nicely formats a storage size using SI units. 46 See Wikipedia and other sources for rationale. 47 Prefixes are k, M, G, ... 48 Sizes are powers of 10. 49 Actual result should be suffixed with bit or byte, not b or B. 50 51 @param units: the unit size to format 52 @type units: int or float 53 @param precision: the number of floating point digits to use 54 @type precision: int 55 56 @rtype: string 57 @returns: value of units, formatted using SI scale and the given precision 58 """ 59 60 # XXX: We might end up calling float(), which breaks 61 # when using LC_NUMERIC when it is not C 62 import locale 63 locale.setlocale(locale.LC_NUMERIC, "C") 64 65 prefixes = ['E', 'P', 'T', 'G', 'M', 'k', ''] 66 67 value = float(units) 68 prefix = prefixes.pop() 69 while prefixes and value >= 1000: 70 prefix = prefixes.pop() 71 value /= 1000 72 73 format = "%%.%df %%s" % precision 74 return format % (value, prefix)
75
76 -def formatTime(seconds, fractional=0):
77 """ 78 Nicely format time in a human-readable format. 79 Will chunks weeks, days, hours and minutes. 80 81 @param seconds: the time in seconds to format. 82 @type seconds: int or float 83 @param fractional: how many digits to show for the fractional part. 84 @type fractional: int 85 86 @rtype: string 87 @returns: a nicely formatted time string. 88 """ 89 chunks = [] 90 91 week = 60 * 60 * 24 * 7 92 weeks = seconds / week 93 seconds %= week 94 95 day = 60 * 60 * 24 96 days = seconds / day 97 seconds %= day 98 99 hour = 60 * 60 100 hours = seconds / hour 101 seconds %= hour 102 103 minute = 60 104 minutes = seconds / minute 105 seconds %= minute 106 107 if weeks > 1: 108 chunks.append('%d weeks' % weeks) 109 elif weeks == 1: 110 chunks.append('1 week') 111 112 if days > 1: 113 chunks.append('%d days' % days) 114 elif days == 1: 115 chunks.append('1 day') 116 117 chunk = '%02d:%02d' % (hours, minutes) 118 if fractional > 0: 119 chunk += ':%0*.*f' % (fractional + 3, fractional, seconds) 120 121 chunks.append(chunk) 122 123 124 return " ".join(chunks)
125
126 -def version(binary):
127 """ 128 Print a version block for the flumotion binaries. 129 130 @arg binary: name of the binary 131 @type binary: string 132 """ 133 134 block = [] 135 block.append("%s %s" % (binary, configure.version)) 136 block.append("part of Flumotion - a streaming media server") 137 block.append("(C) Copyright 2004,2005,2006 Fluendo") 138 return "\n".join(block)
139
140 -def mergeImplements(*classes):
141 """ 142 Merge the __implements__ tuples of the given classes into one tuple. 143 """ 144 if twisted.copyright.version[0] < '2': 145 allYourBase = [] 146 for clazz in classes: 147 allYourBase += getattr(clazz, '__implements__', ()) 148 return tuple(allYourBase) 149 else: 150 allYourBase = () 151 for clazz in classes: 152 try: 153 interfaces = [i for i in clazz.__implemented__] 154 except AttributeError: 155 # with twisted 2.0.1, we get AttributeError with a simple 156 # class C: pass 157 # which does not have C.__implemented__ 158 interfaces = [] 159 for interface in interfaces: 160 allYourBase += (interface,) 161 return allYourBase
162
163 -def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null', 164 directory='/'):
165 ''' 166 This forks the current process into a daemon. 167 The stdin, stdout, and stderr arguments are file names that 168 will be opened and be used to replace the standard file descriptors 169 in sys.stdin, sys.stdout, and sys.stderr. 170 These arguments are optional and default to /dev/null. 171 172 The fork will switch to the given directory. 173 ''' 174 # Redirect standard file descriptors. 175 si = open(stdin, 'r') 176 os.dup2(si.fileno(), sys.stdin.fileno()) 177 try: 178 log.outputToFiles(stdout, stderr) 179 except IOError, e: 180 if e.errno == errno.EACCES: 181 print dir(e) 182 log.error('common', 'Permission denied writing to log file %s.', 183 e.filename) 184 185 # first fork 186 try: 187 pid = os.fork() 188 if pid > 0: 189 sys.exit(0) # exit first parent 190 except OSError, e: 191 sys.stderr.write("Failed to fork: (%d) %s\n" % (e.errno, e.strerror)) 192 sys.exit(1) 193 194 # decouple from parent environment 195 try: 196 os.chdir(directory) 197 except OSError, e: 198 from flumotion.common import errors 199 raise errors.SystemError, "Failed to change directory to %s: %s" % ( 200 directory, e.strerror) 201 os.umask(0) 202 os.setsid() 203 204 # do second fork 205 try: 206 pid = os.fork() 207 if pid > 0: 208 sys.exit(0) # exit second parent 209 except OSError, e: 210 sys.stderr.write("Failed to fork: (%d) %s\n" % (e.errno, e.strerror)) 211 sys.exit(1)
212 213 # Now I am a daemon! 214 # don't add stuff here that can fail, because from now on the program 215 # will keep running regardless of tracebacks 216
217 -def startup(processType, processName, daemonize=False, daemonizeTo='/'):
218 """ 219 Prepare a process for starting, logging appropriate standarised messages. 220 First daemonizes the process, if daemonize is true. 221 """ 222 log.info(processType, "Starting %s '%s'", processType, processName) 223 224 if daemonize: 225 daemonizeHelper(processType, daemonizeTo, processName) 226 227 log.info(processType, "Started %s '%s'", processType, processName) 228 229 def shutdownStarted(): 230 log.info(processType, "Stopping %s '%s'", processType, processName)
231 def shutdownEnded(): 232 log.info(processType, "Stopped %s '%s'", processType, processName) 233 234 # import inside function so we avoid affecting startup 235 from twisted.internet import reactor 236 reactor.addSystemEventTrigger('before', 'shutdown', 237 shutdownStarted) 238 reactor.addSystemEventTrigger('after', 'shutdown', 239 shutdownEnded) 240
241 -def daemonizeHelper(processType, daemonizeTo='/', processName=None):
242 """ 243 Daemonize a process, writing log files and PID files to conventional 244 locations. 245 246 @param processType: The process type, for example 'worker'. Used 247 as part of the log file and PID file names. 248 @type processType: str 249 @param daemonizeTo: The directory that the daemon should run in. 250 @type daemonizeTo: str 251 @param processName: The service name of the process. Used to 252 disambiguate different instances of the same daemon. 253 @type processName: str 254 """ 255 256 ensureDir(configure.logdir, "log file") 257 ensureDir(configure.rundir, "run file") 258 259 pid = getPid(processType, processName) 260 if pid: 261 raise SystemError( 262 "A %s service named '%s' is already running with pid %d" 263 % (processType, processName or processType, pid)) 264 265 log.debug(processType, "%s service named '%s' daemonizing", 266 processType, processName) 267 268 if processName: 269 logPath = os.path.join(configure.logdir, 270 '%s.%s.log' % (processType, processName)) 271 else: 272 logPath = os.path.join(configure.logdir, 273 '%s.log' % (processType,)) 274 log.debug(processType, 'Further logging will be done to %s', logPath) 275 276 # here we daemonize; so we also change our pid 277 daemonize(stdout=logPath, stderr=logPath, directory=daemonizeTo) 278 279 log.debug(processType, 'Started daemon') 280 281 # from now on I should keep running until killed, whatever happens 282 path = writePidFile(processType, processName) 283 log.debug(processType, 'written pid file %s', path) 284 285 # import inside function so we avoid affecting startup 286 from twisted.internet import reactor 287 def _deletePidFile(): 288 log.debug(processType, 'deleting pid file') 289 deletePidFile(processType, processName)
290 reactor.addSystemEventTrigger('after', 'shutdown', 291 _deletePidFile) 292 293
294 -def argRepr(args=(), kwargs={}, max=-1):
295 """ 296 Return a string representing the given args. 297 """ 298 # FIXME: rename function 299 # FIXME: implement max 300 301 # check validity of input 302 assert (type(args) is tuple or 303 type(args) is list) 304 assert type(kwargs) is dict 305 306 s = '' 307 args = list(args) 308 309 if args: 310 args = map(repr, args) 311 s += ', '.join(args) 312 313 if kwargs: 314 r = [(key + '=' + repr(item)) 315 for key, item in kwargs.items()] 316 317 if s: 318 s += ', ' 319 s += ', '.join(r) 320 321 return s
322
323 -def ensureDir(dir, description):
324 """ 325 Ensure the given directory exists, creating it if not. 326 Raises a SystemError if this fails, including the given description. 327 """ 328 if not os.path.exists(dir): 329 try: 330 os.makedirs(dir) 331 except: 332 from flumotion.common import errors 333 raise errors.SystemError, "could not create %s directory %s" % ( 334 description, dir)
335
336 -def getPidPath(type, name=None):
337 """ 338 Get the full path to the pid file for the given process type and name. 339 """ 340 path = os.path.join(configure.rundir, '%s.pid' % type) 341 if name: 342 path = os.path.join(configure.rundir, '%s.%s.pid' % (type, name)) 343 log.debug('common', 'getPidPath for type %s, name %r: %s' % ( 344 type, name, path)) 345 return path
346
347 -def writePidFile(type, name=None):
348 """ 349 Write a pid file in the run directory, using the given process type 350 and process name for the filename. 351 352 @rtype: str 353 @returns: full path to the pid file that was written 354 """ 355 ensureDir(configure.rundir, "rundir") 356 pid = os.getpid() 357 path = getPidPath(type, name) 358 file = open(path, 'w') 359 file.write("%d\n" % pid) 360 file.close() 361 return path
362
363 -def deletePidFile(type, name=None):
364 """ 365 Delete the pid file in the run directory, using the given process type 366 and process name for the filename. 367 368 @rtype: str 369 @returns: full path to the pid file that was written 370 """ 371 path = getPidPath(type, name) 372 os.unlink(path) 373 return path
374
375 -def getPid(type, name=None):
376 """ 377 Get the pid from the pid file in the run directory, using the given 378 process type and process name for the filename. 379 380 @returns: pid of the process, or None if not running or file not found. 381 """ 382 383 pidPath = getPidPath(type, name) 384 log.log('common', 'pidfile for %s %s is %s' % (type, name, pidPath)) 385 if not os.path.exists(pidPath): 386 return 387 388 file = open(pidPath, 'r') 389 pid = file.readline() 390 file.close() 391 if not pid or int(pid) == 0: 392 return 393 394 return int(pid)
395
396 -def termPid(pid):
397 """ 398 Send the given process a TERM signal. 399 400 @returns: whether or not the process with the given pid was running 401 """ 402 try: 403 os.kill(pid, signal.SIGTERM) 404 return True 405 except OSError, e: 406 if not e.errno == errno.ESRCH: 407 # FIXME: unhandled error, maybe give some better info ? 408 raise 409 return False
410
411 -def killPid(pid):
412 """ 413 Send the given process a KILL signal. 414 415 @returns: whether or not the process with the given pid was running 416 """ 417 try: 418 os.kill(pid, signal.SIGKILL) 419 return True 420 except OSError, e: 421 if not e.errno == errno.ESRCH: 422 # FIXME: unhandled error, maybe give some better info ? 423 raise 424 return False
425
426 -def checkPidRunning(pid):
427 """ 428 Check if the given pid is currently running. 429 430 @returns: whether or not a process with that pid is active. 431 """ 432 try: 433 os.kill(pid, 0) 434 return True 435 except OSError, e: 436 if e.errno is not errno.ESRCH: 437 raise 438 return False
439
440 -def waitPidFile(type, name=None):
441 """ 442 Wait for the given process type and name to have started and created 443 a pid file. 444 445 Return the pid. 446 """ 447 # getting it from the start avoids an unneeded time.sleep 448 pid = getPid(type, name) 449 450 while not pid: 451 time.sleep(0.1) 452 pid = getPid(type, name) 453 454 return pid
455
456 -def waitForTerm():
457 """ 458 Wait until we get killed by a TERM signal (from someone else). 459 """ 460 461 class Waiter: 462 def __init__(self): 463 self.sleeping = True 464 import signal 465 self.oldhandler = signal.signal(signal.SIGTERM, 466 self._SIGTERMHandler)
467 468 def _SIGTERMHandler(self, number, frame): 469 self.sleeping = False 470 471 def sleep(self): 472 while self.sleeping: 473 time.sleep(0.1) 474 475 waiter = Waiter() 476 waiter.sleep() 477
478 -def addressGetHost(a):
479 """ 480 Get the host name of an IPv4 address. 481 482 @type a: L{twisted.internet.address.IPv4Address} 483 """ 484 if not isinstance(a, address.IPv4Address) and not isinstance(a, 485 address.UNIXAddress): 486 raise TypeError("object %r is not an IPv4Address or UNIXAddress" % a) 487 if isinstance(a, address.UNIXAddress): 488 return 'localhost' 489 490 try: 491 host = a.host 492 except AttributeError: 493 host = a[1] 494 return host
495
496 -def addressGetPort(a):
497 """ 498 Get the port number of an IPv4 address. 499 500 @type a: L{twisted.internet.address.IPv4Address} 501 """ 502 assert(isinstance(a, address.IPv4Address)) 503 try: 504 port = a.port 505 except AttributeError: 506 port = a[2] 507 return port
508
509 -def componentPath(componentName, parentName):
510 """ 511 Create a path string out of the name of a component and its parent. 512 513 @depreciated: Use @componentId instead 514 """ 515 return '/%s/%s' % (parentName, componentName)
516
517 -def componentId(parentName, componentName):
518 """ 519 Create a C{componentId} based on the C{parentName} and C{componentName}. 520 521 A C{componentId} uniquely identifies a component within a planet. 522 523 @since: 0.3.1 524 525 @rtype: str 526 """ 527 return '/%s/%s' % (parentName, componentName)
528
529 -def parseComponentId(componentId):
530 """ 531 @since: 0.3.1 532 533 @rtype: tuple of (str, str) 534 @return: tuple of (flowName, componentName) 535 """ 536 list = componentId.split("/") 537 assert len(list) == 3 538 assert list[0] == '' 539 return (list[1], list[2])
540
541 -def feedId(componentName, feedName):
542 """ 543 Create a C{feedId} based on the C{componentName} and C{feedName}. 544 545 A C{feedId} uniquely identifies a feed within a flow or atmosphere. 546 It identifies the feed from a feeder to an eater. 547 548 @since: 0.3.1 549 550 @rtype: str 551 """ 552 return "%s:%s" % (componentName, feedName)
553
554 -def parseFeedId(feedId):
555 """ 556 @since: 0.3.1 557 558 @rtype: tuple of (str, str) 559 @return: tuple of (componentName, feedName) 560 """ 561 list = feedId.split(":") 562 assert len(list) == 2, "feedId %s should contain exactly one ':'" % feedId 563 return (list[0], list[1])
564
565 -def fullFeedId(flowName, componentName, feedName):
566 """ 567 Create a C{fullFeedId} based on the C{flowName}, C{componentName} and 568 C{feedName}. 569 570 A C{fullFeedId} uniquely identifies a feed within a planet. 571 572 @since: 0.3.1 573 574 @rtype: str 575 """ 576 return feedId(componentId(flowName, componentName), feedName)
577
578 -def parseFullFeedId(fullFeedId):
579 """ 580 @since: 0.3.1 581 582 @rtype: tuple of (str, str, str) 583 @return: tuple of (flowName, componentName, feedName) 584 """ 585 list = fullFeedId.split(":") 586 assert len(list) == 2 587 flowName, componentName = parseComponentId(list[0]) 588 return (flowName, componentName, list[1])
589
590 -def objRepr(object):
591 """ 592 Return a string giving the fully qualified class of the given object. 593 """ 594 c = object.__class__ 595 return "%s.%s" % (c.__module__, c.__name__)
596
597 -def pathToModuleName(path):
598 """ 599 Convert the given (relative) path to the python module it would have to 600 be imported as. 601 602 Return None if the path is not a valid python module 603 """ 604 # __init__ is last because it works on top of the first three 605 valid = False 606 suffixes = ['.pyc', '.pyo', '.py', os.path.sep + '__init__'] 607 for s in suffixes: 608 if path.endswith(s): 609 path = path[:-len(s)] 610 valid = True 611 612 # if the path still contains dots, it can't possibly be a valid module 613 if not '.' in path: 614 valid = True 615 616 if not valid: 617 return None 618 619 return ".".join(path.split(os.path.sep))
620
621 -def getLL():
622 """ 623 Return the (at most) two-letter language code set for message translation. 624 """ 625 # LANGUAGE is a GNU extension; it can be colon-seperated but we ignore the 626 # advanced stuff. If that's not present, just use LANG, as normal. 627 language = os.environ.get('LANGUAGE', None) 628 if language != None: 629 LL = language[:2] 630 else: 631 lang = os.environ.get('LANG', 'en') 632 LL = lang[:2] 633 634 return LL
635
636 -def gettexter(domain):
637 """ 638 Returns a method you can use as _ to translate strings for the given 639 domain. 640 """ 641 import gettext 642 return lambda s: gettext.dgettext(domain, s)
643
644 -def compareVersions(first, second):
645 """ 646 Compares two version strings. Returns -1, 0 or 1 if first is smaller than, 647 equal to or larger than second. 648 649 @type first: str 650 @type second: str 651 652 @rtype: int 653 """ 654 if first == second: 655 return 0 656 657 firsts = first.split(".") 658 seconds = second.split(".") 659 660 while firsts or seconds: 661 f = 0 662 s = 0 663 try: 664 f = int(firsts[0]) 665 del firsts[0] 666 except IndexError: 667 pass 668 try: 669 s = int(seconds[0]) 670 del seconds[0] 671 except IndexError: 672 pass 673 674 if f < s: 675 return -1 676 if f > s: 677 return 1 678 679 return 0
680
681 -def versionTupleToString(versionTuple):
682 """ 683 Converts a version tuple to a string. If the tuple has a zero nano number, 684 it is dropped from the string. 685 686 @since: 0.4.1 687 688 @type versionTuple: tuple 689 690 @rtype: str 691 """ 692 if len(versionTuple) == 4 and versionTuple[3] == 0: 693 versionTuple = versionTuple[:3] 694 695 return ".".join([str(i) for i in versionTuple])
696
697 -def _uniq(l, key=lambda x: x):
698 """ 699 Filters out duplicate entries in a list. 700 """ 701 out = [] 702 for x in l: 703 if key(x) not in [key(y) for y in out]: 704 out.append(x) 705 return out
706
707 -def _call_each_method(obj, method, mro, args, kwargs):
708 procs = [] 709 for c in mro: 710 if hasattr(c, method): 711 proc = getattr(c, method) 712 assert callable(proc) and hasattr(proc, 'im_func'),\ 713 'attr %s of class %s is not a method' % (method, c) 714 procs.append(proc) 715 716 # In a hierarchy A -> B, if A implements the method, B will inherit 717 # it as well. Compare the functions implementing the methods so as 718 # to avoid calling them twice. 719 procs = _uniq(procs, lambda proc: proc.im_func) 720 721 for proc in procs: 722 proc(obj, *args, **kwargs)
723
724 -def call_each_method(obj, method, *args, **kwargs):
725 """ 726 Invoke all implementations of a method on an object. 727 728 Searches for method implementations in the object's class and all of 729 the class' superclasses. Calls the methods in method resolution 730 order, which goes from subclasses to superclasses. 731 """ 732 mro = type(obj).__mro__ 733 _call_each_method(obj, method, mro, args, kwargs)
734
735 -def call_each_method_reversed(obj, method, *args, **kwargs):
736 """ 737 Invoke all implementations of a method on an object. 738 739 Like call_each_method, but calls the methods in reverse method 740 resolution order, from superclasses to subclasses. 741 """ 742 # do a list() so as to copy the mro, we reverse the list in 743 # place so as to start with the base class 744 mro = list(type(obj).__mro__) 745 mro.reverse() 746 _call_each_method(obj, method, mro, args, kwargs)
747
748 -class InitMixin(object):
749 """ 750 A mixin class to help with object initialization. 751 752 In some class hierarchies, __init__ is only used for initializing 753 instance variables. In these cases it is advantageous to avoid the 754 need to "chain up" to a parent implementation of a method. Adding 755 this class to your hierarchy will, for each class in the object's 756 class hierarchy, call the class's init() implementation on the 757 object. 758 759 Note that the function is called init() without underscrores, and 760 that there is no need to chain up to superclasses' implementations. 761 762 Uses call_each_method_reversed() internally. 763 """ 764
765 - def __init__(self, *args, **kwargs):
766 call_each_method_reversed(self, 'init', *args, **kwargs)
767
768 -def strToBool(string):
769 """ 770 @type string: str 771 772 @return: True if the string represents a value we interpret as true. 773 """ 774 if string in ('True', 'true', '1', 'yes'): 775 return True 776 777 return False
778