Package flumotion :: Package common :: Module config
[hide private]

Source Code for Module flumotion.common.config

  1  # -*- Mode: Python; test-case-name: flumotion.test.test_config -*- 
  2  # vi:si:et:sw=4:sts=4:ts=4 
  3  # 
  4  # Flumotion - a streaming media server 
  5  # Copyright (C) 2004,2005,2006,2007 Fluendo, S.L. (www.fluendo.com). 
  6  # All rights reserved. 
  7   
  8  # This file may be distributed and/or modified under the terms of 
  9  # the GNU General Public License version 2 as published by 
 10  # the Free Software Foundation. 
 11  # This file is distributed without any warranty; without even the implied 
 12  # warranty of merchantability or fitness for a particular purpose. 
 13  # See "LICENSE.GPL" in the source distribution for more information. 
 14   
 15  # Licensees having purchased or holding a valid Flumotion Advanced 
 16  # Streaming Server license may use this file in accordance with the 
 17  # Flumotion Advanced Streaming Server Commercial License Agreement. 
 18  # See "LICENSE.Flumotion" in the source distribution for more information. 
 19   
 20  # Headers in this file shall remain intact. 
 21   
 22  """ 
 23  parsing of configuration files 
 24  """ 
 25   
 26  # Config XML FIXME: Make all <property> nodes be children of 
 27  # <properties>; it's the only thing standing between now and a 
 28  # table-driven, verifying config XML parser 
 29   
 30  import os 
 31  from xml.dom import minidom, Node 
 32  from xml.parsers import expat 
 33   
 34  from twisted.python import reflect  
 35   
 36  from flumotion.common import log, errors, common, registry, fxml 
 37  from flumotion.configure import configure 
 38   
 39  from errors import ConfigError, ComponentWorkerConfigError 
 40   
 41  # all these string values should result in True 
 42  BOOL_TRUE_VALUES = ['True', 'true', '1', 'Yes', 'yes'] 
 43   
44 -class ConfigEntryComponent(log.Loggable):
45 "I represent a <component> entry in a planet config file" 46 nice = 0 47 logCategory = 'config'
48 - def __init__(self, name, parent, type, config, defs, worker):
49 self.name = name 50 self.parent = parent 51 self.type = type 52 self.config = config 53 self.defs = defs 54 self.worker = worker
55
56 - def getType(self):
57 return self.type
58
59 - def getName(self):
60 return self.name
61
62 - def getParent(self):
63 return self.parent
64
65 - def getConfigDict(self):
66 return self.config
67
68 - def getWorker(self):
69 return self.worker
70
71 -class ConfigEntryFlow:
72 "I represent a <flow> entry in a planet config file"
73 - def __init__(self, name):
74 self.name = name 75 self.components = {}
76
77 -class ConfigEntryManager:
78 "I represent a <manager> entry in a planet config file"
79 - def __init__(self, name, host, port, transport, certificate, bouncer, 80 fludebug, plugs):
81 self.name = name 82 self.host = host 83 self.port = port 84 self.transport = transport 85 self.certificate = certificate 86 self.bouncer = bouncer 87 self.fludebug = fludebug 88 self.plugs = plugs
89
90 -class ConfigEntryAtmosphere:
91 "I represent a <atmosphere> entry in a planet config file"
92 - def __init__(self):
93 self.components = {}
94
95 - def __len__(self):
96 return len(self.components)
97
98 -class BaseConfigParser(fxml.Parser):
99 - def __init__(self, file):
100 """ 101 @param file: The file to parse, either as an open file object, 102 or as the name of a file to open. 103 @type file: str or file. 104 """ 105 self.add(file)
106
107 - def add(self, file):
108 """ 109 @param file: The file to parse, either as an open file object, 110 or as the name of a file to open. 111 @type file: str or file. 112 """ 113 try: 114 self.path = os.path.split(file.name)[0] 115 except AttributeError: 116 # for file objects without the name attribute, e.g. StringIO 117 self.path = None 118 119 try: 120 self.doc = self.getRoot(file) 121 except fxml.ParserError, e: 122 raise ConfigError(e.args[0])
123
124 - def getPath(self):
125 return self.path
126
127 - def export(self):
128 return self.doc.toxml()
129
130 - def get_float_values(self, nodes):
131 return [float(subnode.childNodes[0].data) for subnode in nodes]
132
133 - def get_int_values(self, nodes):
134 return [int(subnode.childNodes[0].data) for subnode in nodes]
135
136 - def get_long_values(self, nodes):
137 return [long(subnode.childNodes[0].data) for subnode in nodes]
138
139 - def get_bool_values(self, nodes):
140 return [subnode.childNodes[0].data in BOOL_TRUE_VALUES \ 141 for subnode in nodes]
142
143 - def get_string_values(self, nodes):
144 values = [] 145 for subnode in nodes: 146 try: 147 data = subnode.childNodes[0].data 148 except IndexError: 149 continue 150 # libxml always gives us unicode, even when we encode values 151 # as strings. try to make normal strings again, unless that 152 # isn't possible. 153 try: 154 data = str(data) 155 except UnicodeEncodeError: 156 pass 157 if '\n' in data: 158 parts = [x.strip() for x in data.split('\n')] 159 data = ' '.join(parts) 160 values.append(data) 161 162 return values
163
164 - def get_raw_string_values(self, nodes):
165 values = [] 166 for subnode in nodes: 167 try: 168 data = str(subnode.childNodes[0].data) 169 values.append(data) 170 except IndexError: # happens on a subnode without childNOdes 171 pass 172 173 string = "".join(values) 174 return [string, ]
175
176 - def get_fraction_values(self, nodes):
177 def fraction_from_string(string): 178 parts = string.split('/') 179 if len(parts) == 2: 180 return (int(parts[0]), int(parts[1])) 181 elif len(parts) == 1: 182 return (int(parts[0]), 1) 183 else: 184 raise ConfigError("Invalid fraction: %s", string)
185 return [fraction_from_string(subnode.childNodes[0].data) 186 for subnode in nodes]
187
188 - def parseProperties(self, node, properties, error):
189 """ 190 Parse a <property>-containing node in a configuration XML file. 191 Ignores any subnode not named <property>. 192 193 @param node: The <properties> XML node to parse. 194 @type node: L{xml.dom.Node} 195 @param properties: The set of valid properties. 196 @type properties: list of 197 L{flumotion.common.registry.RegistryEntryProperty} 198 @param error: An exception factory for parsing errors. 199 @type error: Callable that maps str => Exception. 200 201 @returns: The parsed properties, as a dict of name => value. 202 Absent optional properties will not appear in the dict. 203 """ 204 # FIXME: non-validating, see first FIXME 205 # XXX: We might end up calling float(), which breaks 206 # when using LC_NUMERIC when it is not C -- only in python 207 # 2.3 though, no prob in 2.4 208 import locale 209 locale.setlocale(locale.LC_NUMERIC, "C") 210 211 properties_given = {} 212 for subnode in node.childNodes: 213 if subnode.nodeName == 'property': 214 if not subnode.hasAttribute('name'): 215 raise error("<property> must have a name attribute") 216 name = subnode.getAttribute('name') 217 if not name in properties_given: 218 properties_given[name] = [] 219 properties_given[name].append(subnode) 220 221 property_specs = dict([(x.name, x) for x in properties]) 222 223 config = {} 224 for name, nodes in properties_given.items(): 225 if not name in property_specs: 226 raise error("%s: unknown property" % name) 227 228 definition = property_specs[name] 229 230 if not definition.multiple and len(nodes) > 1: 231 raise error("%s: multiple value specified but not " 232 "allowed" % name) 233 234 parsers = {'string': self.get_string_values, 235 'rawstring': self.get_raw_string_values, 236 'int': self.get_int_values, 237 'long': self.get_long_values, 238 'bool': self.get_bool_values, 239 'float': self.get_float_values, 240 'fraction': self.get_fraction_values} 241 242 if not definition.type in parsers: 243 raise error("%s: invalid property type %s" 244 % (name, definition.type)) 245 246 values = parsers[definition.type](nodes) 247 248 if values == []: 249 continue 250 251 if not definition.multiple: 252 values = values[0] 253 254 config[name] = values 255 256 for name, definition in property_specs.items(): 257 if definition.isRequired() and not name in config: 258 raise error("%s: required but unspecified property" 259 % name) 260 261 return config
262
263 - def parsePlug(self, node):
264 # <plug socket=... type=...> 265 # <property> 266 socket, type = self.parseAttributes(node, ('socket', 'type')) 267 268 try: 269 defs = registry.getRegistry().getPlug(type) 270 except KeyError: 271 raise ConfigError("unknown plug type: %s" % type) 272 273 possible_node_names = ['property'] 274 for subnode in node.childNodes: 275 if (subnode.nodeType == Node.COMMENT_NODE 276 or subnode.nodeType == Node.TEXT_NODE): 277 continue 278 elif subnode.nodeName not in possible_node_names: 279 raise ConfigError("Invalid subnode of <plug>: %s" 280 % subnode.nodeName) 281 282 property_specs = defs.getProperties() 283 def err(str): 284 return ConfigError('%s: %s' % (type, str))
285 properties = self.parseProperties(node, property_specs, err) 286 287 return {'type':type, 'socket':socket, 'properties':properties} 288
289 - def parsePlugs(self, node, sockets):
290 # <plugs> 291 # <plug> 292 # returns: dict of socket -> list of plugs 293 # where a plug is 'type'->str, 'socket'->str, 294 # 'properties'->dict of properties 295 plugs = {} 296 for socket in sockets: 297 plugs[socket] = [] 298 def addplug(plug): 299 if plug['socket'] not in sockets: 300 raise ConfigError("Component does not support " 301 "sockets of type %s" % plug['socket']) 302 plugs[plug['socket']].append(plug)
303 304 parsers = {'plug': (self.parsePlug, addplug)} 305 self.parseFromTable(node, parsers) 306 307 return plugs 308 309 # FIXME: rename to PlanetConfigParser or something (should include the 310 # word 'planet' in the name)
311 -class FlumotionConfigXML(BaseConfigParser):
312 """ 313 I represent a planet configuration file for Flumotion. 314 315 @ivar manager: A L{ConfigEntryManager} containing options for the manager 316 section, filled in at construction time. 317 @ivar atmosphere: A L{ConfigEntryAtmosphere}, filled in when parse() is 318 called. 319 @ivar flows: A list of L{ConfigEntryFlow}, filled in when parse() is 320 called. 321 """ 322 logCategory = 'config' 323
324 - def __init__(self, file):
325 BaseConfigParser.__init__(self, file) 326 327 self.flows = [] 328 self.manager = None 329 self.atmosphere = ConfigEntryAtmosphere() 330 331 # We parse without asking for a registry so the registry doesn't 332 # verify before knowing the debug level 333 self.parse(noRegistry=True)
334
335 - def parse(self, noRegistry=False):
336 # <planet> 337 # <manager> 338 # <atmosphere> 339 # <flow> 340 # ... 341 # </planet> 342 343 root = self.doc.documentElement 344 345 if not root.nodeName == 'planet': 346 raise ConfigError("unexpected root node': %s" % root.nodeName) 347 348 for node in root.childNodes: 349 if node.nodeType != Node.ELEMENT_NODE: 350 continue 351 352 if noRegistry and node.nodeName != 'manager': 353 continue 354 355 if node.nodeName == 'atmosphere': 356 entry = self._parseAtmosphere(node) 357 self.atmosphere.components.update(entry) 358 elif node.nodeName == 'flow': 359 entry = self._parseFlow(node) 360 self.flows.append(entry) 361 elif node.nodeName == 'manager': 362 entry = self._parseManager(node, noRegistry) 363 self.manager = entry 364 else: 365 raise ConfigError("unexpected node under 'planet': %s" % node.nodeName)
366
367 - def _parseAtmosphere(self, node):
368 # <atmosphere> 369 # <component> 370 # ... 371 # </atmosphere> 372 373 ret = {} 374 for child in node.childNodes: 375 if (child.nodeType == Node.TEXT_NODE or 376 child.nodeType == Node.COMMENT_NODE): 377 continue 378 379 if child.nodeName == "component": 380 component = self._parseComponent(child, 'atmosphere') 381 else: 382 raise ConfigError("unexpected 'atmosphere' node: %s" % child.nodeName) 383 384 ret[component.name] = component 385 return ret
386
387 - def _parseComponent(self, node, parent, forManager=False):
388 """ 389 Parse a <component></component> block. 390 391 @rtype: L{ConfigEntryComponent} 392 """ 393 # <component name="..." type="..." worker="..."> 394 # <source>* 395 # <property name="name">value</property>* 396 # </component> 397 398 if not node.hasAttribute('name'): 399 raise ConfigError("<component> must have a name attribute") 400 if not node.hasAttribute('type'): 401 raise ConfigError("<component> must have a type attribute") 402 if forManager: 403 if node.hasAttribute('worker'): 404 raise ComponentWorkerConfigError("components in manager" 405 "cannot have workers") 406 else: 407 if (not node.hasAttribute('worker') 408 or not node.getAttribute('worker')): 409 # new since 0.3, give it a different error 410 raise ComponentWorkerConfigError("<component> must have a" 411 " worker attribute") 412 version = None 413 if node.hasAttribute('version'): 414 versionString = node.getAttribute("version") 415 try: 416 versionList = map(int, versionString.split('.')) 417 if len(versionList) == 3: 418 version = tuple(versionList) + (0,) 419 elif len(versionList) == 4: 420 version = tuple(versionList) 421 except: 422 raise ComponentWorkerConfigError("<component> version not" 423 " parseable") 424 425 # If we don't have a version at all, use the current version 426 if not version: 427 version = configure.versionTuple 428 429 type = str(node.getAttribute('type')) 430 name = str(node.getAttribute('name')) 431 if forManager: 432 worker = None 433 else: 434 worker = str(node.getAttribute('worker')) 435 436 # FIXME: flumotion-launch does not define parent, type, or 437 # avatarId. Thus they don't appear to be necessary, like they're 438 # just extra info for the manager or so. Figure out what's going 439 # on with that. Also, -launch treats clock-master differently. 440 config = { 'name': name, 441 'parent': parent, 442 'type': type, 443 'avatarId': common.componentId(parent, name), 444 'version': version 445 } 446 447 try: 448 defs = registry.getRegistry().getComponent(type) 449 except KeyError: 450 raise errors.UnknownComponentError( 451 "unknown component type: %s" % type) 452 453 possible_node_names = ['source', 'clock-master', 'property', 454 'plugs'] 455 for subnode in node.childNodes: 456 if subnode.nodeType == Node.COMMENT_NODE: 457 continue 458 elif subnode.nodeType == Node.TEXT_NODE: 459 # fixme: should check here that the string is empty 460 # should just make a dtd, gah 461 continue 462 elif subnode.nodeName not in possible_node_names: 463 raise ConfigError("Invalid subnode of <component>: %s" 464 % subnode.nodeName) 465 466 # let the component know what its feeds should be called 467 config['feed'] = defs.getFeeders() 468 469 sources = self._parseSources(node, defs) 470 if sources: 471 config['source'] = sources 472 473 config['clock-master'] = self._parseClockMaster(node) 474 config['plugs'] = self._parsePlugs(node, defs.getSockets()) 475 476 properties = defs.getProperties() 477 478 self.debug('Parsing component: %s' % name) 479 def err(str): 480 return ConfigError('%s: %s' % (name, str))
481 config['properties'] = self.parseProperties(node, properties, err) 482 483 # fixme: all of the information except the worker is in the 484 # config dict: why? 485 return ConfigEntryComponent(name, parent, type, config, defs, worker)
486
487 - def _parseFlow(self, node):
488 # <flow name="..."> 489 # <component> 490 # ... 491 # </flow> 492 # "name" cannot be atmosphere or manager 493 494 if not node.hasAttribute('name'): 495 raise ConfigError("<flow> must have a name attribute") 496 497 name = str(node.getAttribute('name')) 498 if name == 'atmosphere': 499 raise ConfigError("<flow> cannot have 'atmosphere' as name") 500 if name == 'manager': 501 raise ConfigError("<flow> cannot have 'manager' as name") 502 503 flow = ConfigEntryFlow(name) 504 505 for child in node.childNodes: 506 if (child.nodeType == Node.TEXT_NODE or 507 child.nodeType == Node.COMMENT_NODE): 508 continue 509 510 if child.nodeName == "component": 511 component = self._parseComponent(child, name) 512 else: 513 raise ConfigError("unexpected 'flow' node: %s" % child.nodeName) 514 515 flow.components[component.name] = component 516 517 # handle master clock selection 518 masters = [x for x in flow.components.values() 519 if x.config['clock-master']] 520 if len(masters) > 1: 521 raise ConfigError("Multiple clock masters in flow %s: %r" 522 % (name, masters)) 523 524 need_sync = [(x.defs.getClockPriority(), x) 525 for x in flow.components.values() 526 if x.defs.getNeedsSynchronization()] 527 need_sync.sort() 528 need_sync = [x[1] for x in need_sync] 529 530 if need_sync: 531 if masters: 532 master = masters[0] 533 else: 534 master = need_sync[-1] 535 536 masterAvatarId = master.config['avatarId'] 537 self.info("Setting %s as clock master" % masterAvatarId) 538 539 for c in need_sync: 540 c.config['clock-master'] = masterAvatarId 541 elif masters: 542 self.info('master clock specified, but no synchronization ' 543 'necessary -- ignoring') 544 masters[0].config['clock-master'] = None 545 546 return flow
547
548 - def _parseManager(self, node, noRegistry=False):
549 # <manager> 550 # <component> 551 # ... 552 # </manager> 553 554 name = None 555 host = None 556 port = None 557 transport = None 558 certificate = None 559 bouncer = None 560 fludebug = None 561 plugs = {} 562 563 manager_sockets = \ 564 ['flumotion.component.plugs.adminaction.AdminAction', 565 'flumotion.component.plugs.lifecycle.ManagerLifecycle', 566 'flumotion.component.plugs.identity.IdentityProvider'] 567 for k in manager_sockets: 568 plugs[k] = [] 569 570 if node.hasAttribute('name'): 571 name = str(node.getAttribute('name')) 572 573 for child in node.childNodes: 574 if (child.nodeType == Node.TEXT_NODE or 575 child.nodeType == Node.COMMENT_NODE): 576 continue 577 578 if child.nodeName == "host": 579 host = self._nodeGetString("host", child) 580 elif child.nodeName == "port": 581 port = self._nodeGetInt("port", child) 582 elif child.nodeName == "transport": 583 transport = self._nodeGetString("transport", child) 584 if not transport in ('tcp', 'ssl'): 585 raise ConfigError("<transport> must be ssl or tcp") 586 elif child.nodeName == "certificate": 587 certificate = self._nodeGetString("certificate", child) 588 elif child.nodeName == "component": 589 if noRegistry: 590 continue 591 592 if bouncer: 593 raise ConfigError( 594 "<manager> section can only have one <component>") 595 bouncer = self._parseComponent(child, 'manager', 596 forManager=True) 597 elif child.nodeName == "plugs": 598 if noRegistry: 599 continue 600 601 for k, v in self._parsePlugs(node, manager_sockets).items(): 602 plugs[k].extend(v) 603 elif child.nodeName == "debug": 604 fludebug = self._nodeGetString("debug", child) 605 else: 606 raise ConfigError("unexpected '%s' node: %s" % ( 607 node.nodeName, child.nodeName)) 608 609 # FIXME: assert that it is a bouncer ! 610 611 return ConfigEntryManager(name, host, port, transport, certificate, 612 bouncer, fludebug, plugs)
613
614 - def _nodeGetInt(self, name, node):
615 try: 616 value = int(node.firstChild.nodeValue) 617 except ValueError: 618 raise ConfigError("<%s> value must be an integer" % name) 619 except AttributeError: 620 raise ConfigError("<%s> value not specified" % name) 621 return value
622
623 - def _nodeGetString(self, name, node):
624 try: 625 value = str(node.firstChild.nodeValue) 626 except AttributeError: 627 raise ConfigError("<%s> value not specified" % name) 628 return value
629
630 - def _parseSources(self, node, defs):
631 # <source>feeding-component:feed-name</source> 632 eaters = dict([(x.getName(), x) for x in defs.getEaters()]) 633 634 nodes = [] 635 for subnode in node.childNodes: 636 if subnode.nodeName == 'source': 637 nodes.append(subnode) 638 strings = self.get_string_values(nodes) 639 640 # at this point we don't support assigning certain sources to 641 # certain eaters -- a problem to fix later. for now take the 642 # union of the properties. 643 required = [x for x in eaters.values() if x.getRequired()] 644 multiple = [x for x in eaters.values() if x.getMultiple()] 645 646 if len(strings) == 0 and required: 647 raise ConfigError("Component %s wants to eat on %s, but no " 648 "source specified" 649 % (node.nodeName, eaters.keys()[0])) 650 elif len(strings) > 1 and not multiple: 651 raise ConfigError("Component %s does not support multiple " 652 "sources feeding %s (%r)" 653 % (node.nodeName, eaters.keys()[0], strings)) 654 655 return strings
656
657 - def _parseClockMaster(self, node):
658 nodes = [] 659 for subnode in node.childNodes: 660 if subnode.nodeName == 'clock-master': 661 nodes.append(subnode) 662 bools = self.get_bool_values(nodes) 663 664 if len(bools) > 1: 665 raise ConfigError("Only one <clock-master> node allowed") 666 667 if bools and bools[0]: 668 return True # will get changed to avatarId in parseFlow 669 else: 670 return None
671
672 - def _parsePlugs(self, node, sockets):
673 plugs = {} 674 for socket in sockets: 675 plugs[socket] = [] 676 for subnode in node.childNodes: 677 if subnode.nodeName == 'plugs': 678 newplugs = self.parsePlugs(subnode, sockets) 679 for socket in sockets: 680 plugs[socket].extend(newplugs[socket]) 681 return plugs
682 683 # FIXME: move to a config base class ?
684 - def getComponentEntries(self):
685 """ 686 Get all component entries from both atmosphere and all flows 687 from the configuration. 688 689 @rtype: dictionary of /parent/name -> L{ConfigEntryComponent} 690 """ 691 entries = {} 692 if self.atmosphere and self.atmosphere.components: 693 for c in self.atmosphere.components.values(): 694 path = common.componentId('atmosphere', c.name) 695 entries[path] = c 696 697 for flowEntry in self.flows: 698 for c in flowEntry.components.values(): 699 path = common.componentId(c.parent, c.name) 700 entries[path] = c 701 702 return entries
703
704 -class AdminConfigParser(BaseConfigParser):
705 """ 706 Admin configuration file parser. 707 """ 708 logCategory = 'config' 709
710 - def __init__(self, sockets, file):
711 """ 712 @param file: The file to parse, either as an open file object, 713 or as the name of a file to open. 714 @type file: str or file. 715 """ 716 self.plugs = {} 717 for socket in sockets: 718 self.plugs[socket] = [] 719 720 # will start the parse via self.add() 721 BaseConfigParser.__init__(self, file)
722
723 - def _parse(self):
724 # <admin> 725 # <plugs> 726 root = self.doc.documentElement 727 if not root.nodeName == 'admin': 728 raise ConfigError("unexpected root node': %s" % root.nodeName) 729 730 def parseplugs(node): 731 return self.parsePlugs(node, self.plugs.keys())
732 def addplugs(plugs): 733 for socket in plugs: 734 try: 735 self.plugs[socket].extend(plugs[socket]) 736 except KeyError: 737 raise ConfigError("Admin does not support " 738 "sockets of type %s" % socket)
739 parsers = {'plugs': (parseplugs, addplugs)} 740 741 self.parseFromTable(root, parsers) 742
743 - def add(self, file):
744 """ 745 @param file: The file to parse, either as an open file object, 746 or as the name of a file to open. 747 @type file: str or file. 748 """ 749 BaseConfigParser.add(self, file) 750 self._parse()
751