1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 parsing of manager configuration files
24 """
25
26 from flumotion.common import log, errors, common, registry
27 from flumotion.common import config as fluconfig
28 from flumotion.common.xmlwriter import cmpComponentType, XMLWriter
29 from flumotion.configure import configure
30
31 __version__ = "$Rev: 6981 $"
32
33
36
38 def parseFeedId(feedId):
39 if feedId.find(':') == -1:
40 return "%s:default" % feedId
41 else:
42 return feedId
43
44 eaterConfig = conf.get('eater', {})
45 sourceConfig = conf.get('source', [])
46 if eaterConfig == {} and sourceConfig != []:
47 eaters = registry.getRegistry().getComponent(
48 conf.get('type')).getEaters()
49 eatersDict = {}
50 eatersTuple = [(None, parseFeedId(s)) for s in sourceConfig]
51 eatersDict = buildEatersDict(eatersTuple, eaters)
52 conf['eater'] = eatersDict
53
54 if sourceConfig:
55 sources = []
56 for s in sourceConfig:
57 sources.append(parseFeedId(s))
58 conf['source'] = sources
59
61 eaters = dict(conf.get('eater', {}))
62 concat = lambda lists: reduce(list.__add__, lists, [])
63 if not reduce(lambda x,y: y and isinstance(x, tuple),
64 concat(eaters.values()),
65 True):
66 for eater in eaters:
67 aliases = []
68 feeders = eaters[eater]
69 for i in range(len(feeders)):
70 val = feeders[i]
71 if isinstance(val, tuple):
72 feedId, alias = val
73 aliases.append(val[1])
74 else:
75 feedId = val
76 alias = eater
77 while alias in aliases:
78 log.warning('config', "Duplicate alias %s for "
79 "eater %s, uniquifying", alias, eater)
80 alias += '-bis'
81 aliases.append(alias)
82 feeders[i] = (feedId, val)
83 conf['eater'] = eaters
84
85 UPGRADERS = [upgradeEaters, upgradeAliases]
86 CURRENT_VERSION = len(UPGRADERS)
87
89 """Build a eaters dict suitable for forming part of a component
90 config.
91
92 @param eatersList: List of eaters. For example,
93 [('default', 'othercomp:feeder', 'foo')] says
94 that our eater 'default' will be fed by the feed
95 identified by the feedId 'othercomp:feeder', and
96 that it has the alias 'foo'. Alias is optional.
97 @type eatersList: List of (eaterName, feedId, eaterAlias?)
98 @param eaterDefs: The set of allowed and required eaters
99 @type eaterDefs: List of
100 L{flumotion.common.registry.RegistryEntryEater}
101 @returns: Dict of eaterName => [(feedId, eaterAlias)]
102 """
103 def parseEaterTuple(tup):
104 def parse(eaterName, feedId, eaterAlias=None):
105 if eaterAlias is None:
106 eaterAlias = eaterName
107 return (eaterName, feedId, eaterAlias)
108 return parse(*tup)
109
110 eaters = {}
111 for eater, feedId, alias in [parseEaterTuple(t) for t in eatersList]:
112 if eater is None:
113 if not eaterDefs:
114 raise errors.ConfigError(
115 "Feed %r cannot be connected, component has no eaters" %
116 (feedId,))
117
118 eater = eaterDefs[0].getName()
119 if alias is None:
120 alias = eater
121 feeders = eaters.get(eater, [])
122 if feedId in feeders:
123 raise errors.ConfigError("Already have a feedId %s eating from %s" %
124 (feedId, eater))
125 while alias in [a for f, a in feeders]:
126 log.warning('config', "Duplicate alias %s for eater %s, "
127 "uniquifying", alias, eater)
128 alias += '-bis'
129
130 feeders.append((feedId, alias))
131 eaters[eater] = feeders
132 for e in eaterDefs:
133 eater = e.getName()
134 if e.getRequired() and not eater in eaters:
135 raise errors.ConfigError("Component wants to eat on %s,"
136 " but no feeders specified."
137 % (e.getName(),))
138 if not e.getMultiple() and len(eaters.get(eater, [])) > 1:
139 raise errors.ConfigError("Component does not support multiple "
140 "sources feeding %s (%r)"
141 % (eater, eaters[eater]))
142 aliases = reduce(list.__add__,
143 [[x[1] for x in tups] for tups in eaters.values()],
144 [])
145
146
147 while aliases:
148 alias = aliases.pop()
149 if alias in aliases:
150 raise errors.ConfigError("Duplicate alias: %s" % (alias,))
151
152 return eaters
153
154
156 """Build a virtual feeds dict suitable for forming part of a
157 component config.
158
159 @param feedPairs: List of virtual feeds, as name-feederName pairs. For
160 example, [('bar:baz', 'qux')] defines one
161 virtual feed 'bar:baz', which is provided by
162 the component's 'qux' feed.
163 @type feedPairs: List of (feedId, feedName) -- both strings.
164 @param feeders: The feeders exported by this component, from the
165 registry.
166 @type feeders: List of str.
167 """
168 ret = {}
169 for virtual, real in feedPairs:
170 if real not in feeders:
171 raise errors.ConfigError('virtual feed maps to unknown feeder: '
172 '%s -> %s' % (virtual, real))
173 try:
174 common.parseFeedId(virtual)
175 except:
176 raise errors.ConfigError('virtual feed name not a valid feedId: %s'
177 % (virtual,))
178 ret[virtual] = real
179 return ret
180
181 -def dictDiff(old, new, onlyOld=None, onlyNew=None, diff=None,
182 keyBase=None):
183 """Compute the difference between two config dicts.
184
185 @returns: 3 tuple: (onlyOld, onlyNew, diff) where:
186 onlyOld is a list of (key, value), representing key-value
187 pairs that are only in old;
188 onlyNew is a list of (key, value), representing key-value
189 pairs that are only in new;
190 diff is a list of (key, oldValue, newValue), representing
191 keys with different values in old and new; and
192 key is a tuple of strings representing the recursive key
193 to get to a value. For example, ('foo', 'bar') represents
194 the value d['foo']['bar'] on a dict d.
195 """
196
197
198 if onlyOld is None:
199 onlyOld = []
200 onlyNew = []
201 diff = []
202 keyBase = ()
203
204 for k in old:
205 key = (keyBase + (k,))
206 if k not in new:
207 onlyOld.append((key, old[k]))
208 elif old[k] != new[k]:
209 if isinstance(old[k], dict) and isinstance(new[k], dict):
210 dictDiff(old[k], new[k], onlyOld, onlyNew, diff, key)
211 else:
212 diff.append((key, old[k], new[k]))
213
214 for k in new:
215 key = (keyBase + (k,))
216 if k not in old:
217 onlyNew.append((key, new[k]))
218
219 return onlyOld, onlyNew, diff
220
223 def ref(label, k):
224 return "%s%s: '%s'" % (label,
225 ''.join(["[%r]" % (subk,)
226 for subk in k[:-1]]),
227 k[-1])
228
229 out = []
230 for k, v in old:
231 out.append('Only in %s = %r' % (ref(oldLabel, k), v))
232 for k, v in new:
233 out.append('Only in %s = %r' % (ref(newLabel, k), v))
234 for k, oldv, newv in diff:
235 out.append('Value mismatch:')
236 out.append(' %s = %r' % (ref(oldLabel, k), oldv))
237 out.append(' %s = %r' % (ref(newLabel, k), newv))
238 return '\n'.join(out)
239
240 -class ConfigEntryComponent(log.Loggable):
241 "I represent a <component> entry in a planet config file"
242 nice = 0
243 logCategory = 'config'
244
245 __pychecker__ = 'maxargs=13'
246
247 - def __init__(self, name, parent, type, label, propertyList, plugList,
248 worker, eatersList, isClockMaster, project, version,
249 virtualFeeds=None):
250 self.name = name
251 self.parent = parent
252 self.type = type
253 self.label = label
254 self.worker = worker
255 self.defs = registry.getRegistry().getComponent(self.type)
256 try:
257 self.config = self._buildConfig(propertyList, plugList,
258 eatersList, isClockMaster,
259 project, version,
260 virtualFeeds)
261 except errors.ConfigError, e:
262
263 e.args = ("While parsing component %s: %s"
264 % (name, log.getExceptionMessage(e)),)
265 raise
266
267 - def _buildVersionTuple(self, version):
268 if version is None:
269 return configure.versionTuple
270 elif isinstance(version, tuple):
271 assert len(version) == 4
272 return version
273 elif isinstance(version, str):
274 try:
275 def parse(maj, min, mic, nan=0):
276 return maj, min, mic, nan
277 return parse(*map(int, version.split('.')))
278 except:
279 raise errors.ConfigError("<component> version not parseable")
280 raise errors.ConfigError("<component> version not parseable")
281
282 - def _buildConfig(self, propertyList, plugsList, eatersList,
283 isClockMaster, project, version, virtualFeeds):
284 """
285 Build a component configuration dictionary.
286 """
287
288
289
290 config = {'name': self.name,
291 'parent': self.parent,
292 'type': self.type,
293 'config-version': CURRENT_VERSION,
294 'avatarId': common.componentId(self.parent, self.name),
295 'project': project or configure.PACKAGE,
296 'version': self._buildVersionTuple(version),
297 'clock-master': isClockMaster or None,
298 'feed': self.defs.getFeeders(),
299 'properties': fluconfig.buildPropertyDict(propertyList,
300 self.defs.getProperties()),
301 'plugs': fluconfig.buildPlugsSet(plugsList,
302 self.defs.getSockets()),
303 'eater': buildEatersDict(eatersList,
304 self.defs.getEaters()),
305 'source': [tup[1] for tup in eatersList],
306 'virtual-feeds': buildVirtualFeeds(virtualFeeds or [],
307 self.defs.getFeeders())}
308
309 if self.label:
310
311 config['label'] = self.label
312
313 if not config['source']:
314
315 del config['source']
316
317
318 return config
319
322
323 - def getLabel(self):
325
328
329 - def getParent(self):
331
332 - def getConfigDict(self):
334
335 - def getWorker(self):
337
339 "I represent a <flow> entry in a planet config file"
340 - def __init__(self, name, components):
341 self.name = name
342 self.components = {}
343 for c in components:
344 if c.name in self.components:
345 raise errors.ConfigError(
346 'flow %s already has component named %s' % (name, c.name))
347 self.components[c.name] = c
348
350 "I represent a <manager> entry in a planet config file"
351 - def __init__(self, name, host, port, transport, certificate, bouncer,
352 fludebug, plugs):
353 self.name = name
354 self.host = host
355 self.port = port
356 self.transport = transport
357 self.certificate = certificate
358 self.bouncer = bouncer
359 self.fludebug = fludebug
360 self.plugs = plugs
361
363 "I represent a <atmosphere> entry in a planet config file"
364 - def __init__(self):
366
368 return len(self.components)
369
371 """
372 This is a base class for parsing planet configuration files (both manager
373 and flow files).
374 """
375 logCategory = 'config'
376
378 if feedId.find(':') == -1:
379 return "%s:default" % feedId
380 else:
381 return feedId
382
389
390 - def parseComponent(self, node, parent, isFeedComponent,
391 needsWorker):
392 """
393 Parse a <component></component> block.
394
395 @rtype: L{ConfigEntryComponent}
396 """
397
398
399
400
401
402
403
404
405
406
407 attrs = self.parseAttributes(node, ('name', 'type'),
408 ('label', 'worker', 'project', 'version',))
409 name, componentType, label, worker, project, version = attrs
410 if needsWorker and not worker:
411 raise errors.ConfigError('component %s does not specify the worker '
412 'that it is to run on' % (name,))
413 elif worker and not needsWorker:
414 raise errors.ConfigError('component %s specifies a worker to run '
415 'on, but does not need a worker' % (name,))
416
417 properties = []
418 plugs = []
419 eaters = []
420 clockmasters = []
421 sources = []
422 virtual_feeds = []
423
424 def parseBool(node):
425 return self.parseTextNode(node, common.strToBool)
426 parsers = {'property': (self._parseProperty, properties.append),
427 'compound-property': (self._parseCompoundProperty,
428 properties.append),
429 'plugs': (self.parsePlugs, plugs.extend)}
430
431 if isFeedComponent:
432 parsers.update({'eater': (self._parseEater, eaters.extend),
433 'clock-master': (parseBool, clockmasters.append),
434 'source': (self._parseSource, sources.append),
435 'virtual-feed': (self._parseVirtualFeed,
436 virtual_feeds.append)})
437
438 self.parseFromTable(node, parsers)
439
440 if len(clockmasters) == 0:
441 isClockMaster = None
442 elif len(clockmasters) == 1:
443 isClockMaster = clockmasters[0]
444 else:
445 raise errors.ConfigError("Only one <clock-master> node allowed")
446
447 for feedId in sources:
448
449 eaters.append((None, feedId))
450
451 return ConfigEntryComponent(name, parent, componentType, label,
452 properties, plugs, worker, eaters,
453 isClockMaster, project, version,
454 virtual_feeds)
455
458
463
465
466
467
468 name, = self.parseAttributes(node, ('name',))
469 feeds = []
470 parsers = {'feed': (self._parseFeed, feeds.append)}
471 self.parseFromTable(node, parsers)
472 if len(feeds) == 0:
473
474 raise errors.ConfigError(
475 "Eater node %s with no <feed> nodes, is not allowed" % (name,))
476 return [(name, feedId, alias) for feedId, alias in feeds]
477
479 """
480 I represent a planet configuration file for Flumotion.
481
482 @ivar atmosphere: A L{ConfigEntryAtmosphere}, filled in when parse() is
483 called.
484 @ivar flows: A list of L{ConfigEntryFlow}, filled in when parse() is
485 called.
486 """
487 logCategory = 'config'
488
494
514
515
517
518
519
520
521 ret = {}
522 def parseComponent(node):
523 return self.parseComponent(node, 'atmosphere', False, True)
524 def gotComponent(comp):
525 ret[comp.name] = comp
526 parsers = {'component': (parseComponent, gotComponent)}
527 self.parseFromTable(node, parsers)
528 return ret
529
545 parsers = {'component': (parseComponent, components.append)}
546 self.parseFromTable(node, parsers)
547
548
549
550
551 masters = [x for x in components if x.config['clock-master']]
552 if len(masters) > 1:
553 raise errors.ConfigError("Multiple clock masters in flow %s: %s"
554 % (name, ', '.join([m.name for m in masters])))
555
556 need_sync = [(x.defs.getClockPriority(), x) for x in components
557 if x.defs.getNeedsSynchronization()]
558 need_sync.sort()
559 need_sync = [x[1] for x in need_sync]
560
561 if need_sync:
562 if masters:
563 master = masters[0]
564 else:
565 master = need_sync[-1]
566
567 masterAvatarId = master.config['avatarId']
568 self.info("Setting %s as clock master" % masterAvatarId)
569
570 for c in need_sync:
571 c.config['clock-master'] = masterAvatarId
572 elif masters:
573 self.info('master clock specified, but no synchronization '
574 'necessary -- ignoring')
575 masters[0].config['clock-master'] = None
576
577 return ConfigEntryFlow(name, components)
578
579
581 """
582 Get all component entries from both atmosphere and all flows
583 from the configuration.
584
585 @rtype: dictionary of /parent/name -> L{ConfigEntryComponent}
586 """
587 entries = {}
588 if self.atmosphere and self.atmosphere.components:
589 for c in self.atmosphere.components.values():
590 path = common.componentId('atmosphere', c.name)
591 entries[path] = c
592
593 for flowEntry in self.flows:
594 for c in flowEntry.components.values():
595 path = common.componentId(c.parent, c.name)
596 entries[path] = c
597
598 return entries
599
600
601
602
604 """
605 I parse manager configuration out of a planet configuration file.
606
607 @ivar manager: A L{ConfigEntryManager} containing options for the manager
608 section, filled in at construction time.
609 """
610 logCategory = 'config'
611
612 MANAGER_SOCKETS = \
613 ['flumotion.component.plugs.adminaction.AdminAction',
614 'flumotion.component.plugs.lifecycle.ManagerLifecycle',
615 'flumotion.component.plugs.identity.IdentityProvider']
616
631
643
645
646
647 name, = self.parseAttributes(node, (), ('name',))
648 ret = ConfigEntryManager(name, None, None, None, None, None,
649 None, self.plugs)
650
651 def simpleparse(proc):
652 return lambda node: self.parseTextNode(node, proc)
653 def recordval(k):
654 def record(v):
655 if getattr(ret, k):
656 raise errors.ConfigError('duplicate %s: %s'
657 % (k, getattr(ret, k)))
658 setattr(ret, k, v)
659 return record
660 def enum(*allowed):
661 def eparse(v):
662 v = str(v)
663 if v not in allowed:
664 raise errors.ConfigError('unknown value %s (should be '
665 'one of %r)' % (v, allowed))
666 return v
667 return eparse
668
669 parsers = {'host': (simpleparse(str), recordval('host')),
670 'port': (simpleparse(int), recordval('port')),
671 'transport': (simpleparse(enum('tcp', 'ssl')),
672 recordval('transport')),
673 'certificate': (simpleparse(str), recordval('certificate')),
674 'component': (_ignore, _ignore),
675 'plugs': (_ignore, _ignore),
676 'debug': (simpleparse(str), recordval('fludebug'))}
677 self.parseFromTable(node, parsers)
678 return ret
679
681 def parsecomponent(node):
682 return self.parseComponent(node, 'manager', False, False)
683 def gotcomponent(val):
684 if self.bouncer is not None:
685 raise errors.ConfigError('can only have one bouncer '
686 '(%s is superfluous)' % (val.name,))
687
688 self.bouncer = val
689 def parseplugs(node):
690 return fluconfig.buildPlugsSet(self.parsePlugs(node),
691 self.MANAGER_SOCKETS)
692 def gotplugs(newplugs):
693 for socket in self.plugs:
694 self.plugs[socket].extend(newplugs[socket])
695
696 parsers = {'host': (_ignore, _ignore),
697 'port': (_ignore, _ignore),
698 'transport': (_ignore, _ignore),
699 'certificate': (_ignore, _ignore),
700 'component': (parsecomponent, gotcomponent),
701 'plugs': (parseplugs, gotplugs),
702 'debug': (_ignore, _ignore)}
703 self.parseFromTable(node, parsers)
704
720
724
725
761
763 config = component.get('config')
764 attrs = [('name', component.get('name')),
765 ('type', component.get('type')),
766 ('label', config.get('label', component.get('name'))),
767 ('worker', component.get('workerRequested')),
768 ('project', config['project']),
769 ('version', common.versionTupleToString(config['version']))]
770 self.pushTag('component', attrs)
771 for name, feeders in config['eater'].items():
772 self._writeEater(name, feeders)
773 self._writeProperties(config['properties'].items())
774 if isFeedComponent:
775 if config['clock-master'] == config['avatarId']:
776 value = 'true'
777 else:
778 value = 'false'
779 self.writeTag('clock-master', data=value)
780 self._writePlugs(config['plugs'].items())
781 self._writeVirtualFeeds(config['virtual-feeds'].items())
782 self.popTag()
783
785 attrs = [('name', name)]
786 self.pushTag('eater', attrs)
787 for feedId, alias in feeders:
788 attrs = [('alias', alias)]
789 self.writeTag('feed', attrs, feedId)
790 self.popTag()
791
793 def serialise(propVal):
794 if isinstance(propVal, tuple):
795 return "%d/%d" % propVal
796 return propVal
797 for name, value in properties:
798 attrs = [('name', name)]
799 self.writeTag('property', attrs, serialise(value))
800
807
814
816 for name, real in virtualfeeds:
817 attrs = [('name', name),
818 ('real', real)]
819 self.writeTag('virtual-feed', attrs)
820
824