diff --git a/octodns/manager.py b/octodns/manager.py index fc05810..9116742 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -121,6 +121,25 @@ class Manager(object): raise ManagerException('Incorrect provider config for {}' .format(provider_name)) + self.processors = {} + for processor_name, processor_config in \ + self.config.get('processors', {}).items(): + try: + _class = processor_config.pop('class') + except KeyError: + self.log.exception('Invalid processor class') + raise ManagerException('Processor {} is missing class' + .format(processor_name)) + _class = self._get_named_class('processor', _class) + kwargs = self._build_kwargs(processor_config) + try: + self.processors[processor_name] = _class(processor_name, + **kwargs) + except TypeError: + self.log.exception('Invalid processor config') + raise ManagerException('Incorrect processor config for {}' + .format(processor_name)) + zone_tree = {} # sort by reversed strings so that parent zones always come first for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): @@ -222,8 +241,8 @@ class Manager(object): self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) - def _populate_and_plan(self, zone_name, sources, targets, desired=None, - lenient=False): + def _populate_and_plan(self, zone_name, processors, sources, targets, + desired=None, lenient=False): self.log.debug('sync: populating, zone=%s, lenient=%s', zone_name, lenient) @@ -248,6 +267,10 @@ class Manager(object): 'param', source.__class__.__name__) source.populate(zone) + self.log.debug('sync: processing, zone=%s', zone_name) + for processor in processors: + zone = processor.process(zone) + self.log.debug('sync: planning, zone=%s', zone_name) plans = [] @@ -259,7 +282,9 @@ class Manager(object): 'value': 'provider={}'.format(target.id) }) zone.add_record(meta, replace=True) - plan = target.plan(zone) + # TODO: if someone has overrriden plan already this will be a + # breaking change so we probably need to try both ways + plan = target.plan(zone, processors=processors) if plan: plans.append((target, plan)) @@ -315,6 +340,8 @@ class Manager(object): raise ManagerException('Zone {} is missing targets' .format(zone_name)) + processors = config.get('processors', []) + if (eligible_sources and not [s for s in sources if s in eligible_sources]): self.log.info('sync: no eligible sources, skipping') @@ -332,6 +359,15 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) + try: + collected = [] + for processor in processors: + collected.append(self.processors[processor]) + processors = collected + except KeyError: + raise ManagerException('Zone {}, unknown processor: {}' + .format(zone_name, processor)) + try: # rather than using a list comprehension, we break this loop # out so that the `except` block below can reference the @@ -358,8 +394,9 @@ class Manager(object): .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, - zone_name, sources, - targets, lenient=lenient)) + zone_name, processors, + sources, targets, + lenient=lenient)) # Wait on all results and unpack/flatten the plans and store the # desired states in case we need them below @@ -378,6 +415,7 @@ class Manager(object): futures.append(self._executor.submit( self._populate_and_plan, zone_name, + processors, [], [self.providers[t] for t in source_config['targets']], desired=desired[zone_source], @@ -518,6 +556,9 @@ class Manager(object): if isinstance(source, YamlProvider): source.populate(zone) + # TODO: validate + # processors = config.get('processors', []) + def get_zone(self, zone_name): if not zone_name[-1] == '.': raise ManagerException('Invalid zone name {}, missing ending dot' diff --git a/octodns/processors/__init__.py b/octodns/processors/__init__.py new file mode 100644 index 0000000..9431e6a --- /dev/null +++ b/octodns/processors/__init__.py @@ -0,0 +1,17 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from ..zone import Zone + + +class BaseProcessor(object): + + def __init__(self, name): + self.name = name + + def _create_zone(self, zone): + return Zone(zone.name, sub_zones=zone.sub_zones) diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py new file mode 100644 index 0000000..d483ab7 --- /dev/null +++ b/octodns/processors/filters.py @@ -0,0 +1,38 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from . import BaseProcessor + + +class TypeAllowlistFilter(BaseProcessor): + + def __init__(self, name, allowlist): + super(TypeAllowlistFilter, self).__init__(name) + self.allowlist = allowlist + + def process(self, zone): + ret = self._create_zone(zone) + for record in zone.records: + if record._type in self.allowlist: + ret.add_record(record) + + return ret + + +class TypeRejectlistFilter(BaseProcessor): + + def __init__(self, name, rejectlist): + super(TypeRejectlistFilter, self).__init__(name) + self.rejectlist = rejectlist + + def process(self, zone): + ret = self._create_zone(zone) + for record in zone.records: + if record._type not in self.rejectlist: + ret.add_record(record) + + return ret diff --git a/octodns/provider/base.py b/octodns/provider/base.py index ae87844..2a4ab11 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -44,7 +44,7 @@ class BaseProvider(BaseSource): ''' return [] - def plan(self, desired): + def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) existing = Zone(desired.name, desired.sub_zones) @@ -55,6 +55,9 @@ class BaseProvider(BaseSource): self.log.warn('Provider %s used in target mode did not return ' 'exists', self.id) + for processor in processors: + existing = processor.process(existing) + # compute the changes at the zone/record level changes = existing.changes(desired, self) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index dc047e8..3a09809 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -352,7 +352,7 @@ class TestManager(TestCase): pass # This should be ok, we'll fall back to not passing it - manager._populate_and_plan('unit.tests.', [NoLenient()], []) + manager._populate_and_plan('unit.tests.', [], [NoLenient()], []) class NoZone(SimpleProvider):