From 90cc9576f540e43de4b7ac5d7bfee02a87ea5f58 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 11 Jun 2020 17:46:29 -0400 Subject: [PATCH 01/14] Increase Cloudflare page size Increase Cloudflare page size to reduce request count `GET zones` has a MAX of 50 and a default of 20 https://api.cloudflare.com/#zone-list-zones `GET zones/:zone_identifier/dns_records` has a MAX of 100 and a default of 20 https://api.cloudflare.com/#dns-records-for-a-zone-list-dns-records --- octodns/provider/cloudflare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 698fbee..96febf4 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -142,7 +142,7 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page}) + params={'page': page, 'per_page': 50}) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -251,7 +251,7 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page}) + resp = self._try_request('GET', path, params={'page': page, 'per_page': 100}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: From b80d1575e6d5268e3ee69c14adcc8e2ad9dcfa88 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 11 Jun 2020 17:56:31 -0400 Subject: [PATCH 02/14] Update tests with new per_page params --- octodns/provider/cloudflare.py | 3 ++- tests/test_octodns_provider_cloudflare.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 96febf4..96c9f5e 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -251,7 +251,8 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page, 'per_page': 100}) + resp = self._try_request('GET', path, params={'page': page, + 'per_page': 100}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 08608ea..735d95c 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -426,7 +426,7 @@ class TestCloudflareProvider(TestCase): # get the list of zones, create a zone, add some records, update # something, and delete something provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -531,7 +531,7 @@ class TestCloudflareProvider(TestCase): # Get zones, create zone, create a record, delete a record provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -1302,7 +1302,8 @@ class TestCloudflareProvider(TestCase): provider._request.side_effect = [result] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # One retry required provider._zones = None @@ -1313,7 +1314,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # Two retries required provider._zones = None @@ -1325,7 +1327,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # # Exhaust our retries provider._zones = None From bac16622426d21768ebf6b4c895936965a3ffd66 Mon Sep 17 00:00:00 2001 From: DavHau Date: Mon, 15 Jun 2020 16:51:28 +0000 Subject: [PATCH 03/14] fix: dependency 'ipaddress' unnecessary for py >= 3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c56aa82..142b209 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', - 'ipaddress>=1.0.22', + 'ipaddress>=1.0.22; python_version<"3.2"', 'natsort>=5.5.0', 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2', From 84048dbde9df4de639141d32ca4722c538b2b92b Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Mon, 22 Jun 2020 17:27:41 -0400 Subject: [PATCH 04/14] Cloudflare: Make page size configurable --- octodns/provider/cloudflare.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 96c9f5e..db937e5 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -58,6 +58,10 @@ class CloudflareProvider(BaseProvider): retry_count: 4 # Optional. Default: 300. Number of seconds to wait before retrying. retry_period: 300 + # Optional. Default: 50. Number of zones per page. + zones_per_page: 50 + # Optional. Default: 100. Number of dns records per page. + records_per_page: 100 Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed via the YAML provider like so: @@ -78,7 +82,8 @@ class CloudflareProvider(BaseProvider): TIMEOUT = 15 def __init__(self, id, email=None, token=None, cdn=False, retry_count=4, - retry_period=300, *args, **kwargs): + retry_period=300, zones_per_page=50, records_per_page=100, + *args, **kwargs): self.log = getLogger('CloudflareProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id, email, cdn) @@ -99,6 +104,8 @@ class CloudflareProvider(BaseProvider): self.cdn = cdn self.retry_count = retry_count self.retry_period = retry_period + self.zones_per_page = zones_per_page + self.records_per_page = records_per_page self._sess = sess self._zones = None @@ -142,7 +149,10 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page, 'per_page': 50}) + params={ + 'page': page, + 'per_page': self.zones_per_page + }) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -252,7 +262,7 @@ class CloudflareProvider(BaseProvider): page = 1 while page: resp = self._try_request('GET', path, params={'page': page, - 'per_page': 100}) + 'per_page': self.records_per_page}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: From c2f541546b7818c8a2acbafbed2c6a13855fe290 Mon Sep 17 00:00:00 2001 From: John Dale Date: Thu, 9 Jul 2020 03:47:11 +0000 Subject: [PATCH 05/14] Adding Octodns provider class for easyDNS This provider class for easydns.com adds support for basic dns records through the easyDNS v3 API. Support for dynamic and geo based dns records is planned for a future update. Sample configuration for the easyDNS provider are: easydns: class: octodns.provider.easydns.EasyDNSProvider token: apikey: The token and key values are found on the easyDNS customer portal at: https://cp.easydns.com/manage/security/api/production_info.php Also, below are some optional configuration parameters which can be added to override the class defaults. By default the provider class connects with the LIVE easyDNS API, if you wish to perform testing with the easyDNS Sandbox API you can enable it by adding the following configuration parameter: sandbox: True Note, the API token and key are different for the sandbox than they are for the production API, you can obtain sandbox credentials at: https://cp.easydns.com/manage/security/api/sandbox_info.php Lastly, if you have created Domain Portfolios through the easyDNS CP you can configure which portfolio new domains will be added to by supplying the portfolio option with the name of your portfolio. portfolio: --- README.md | 5 +- octodns/provider/easydns.py | 453 +++++++++++++++++++++++++ tests/fixtures/easydns-records.json | 274 +++++++++++++++ tests/test_octodns_provider_easydns.py | 449 ++++++++++++++++++++++++ 4 files changed, 1179 insertions(+), 2 deletions(-) create mode 100644 octodns/provider/easydns.py create mode 100644 tests/fixtures/easydns-records.json create mode 100644 tests/test_octodns_provider_easydns.py diff --git a/README.md b/README.md index ce9be86..0cc2ac7 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | | +| [EasyDNSProvider](/octodns/provider/easydns.py) | | A, AAAA, CAA, CNAME, MX, NAPTR, NS, SRV, TXT | No | | | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | @@ -283,8 +284,8 @@ OctoDNS is licensed under the [MIT license](LICENSE). The MIT license grant is not for GitHub's trademarks, which include the logo designs. GitHub reserves all trademark and copyright rights in and to all GitHub trademarks. GitHub's logos include, for instance, the stylized designs that include "logo" in the file title in the following folder: https://github.com/github/octodns/tree/master/docs/logos/ -GitHub® and its stylized versions and the Invertocat mark are GitHub's Trademarks or registered Trademarks. When using GitHub's logos, be sure to follow the GitHub logo guidelines. +GitHub® and its stylized versions and the Invertocat mark are GitHub's Trademarks or registered Trademarks. When using GitHub's logos, be sure to follow the GitHub logo guidelines. ## Authors -OctoDNS was designed and authored by [Ross McFarland](https://github.com/ross) and [Joe Williams](https://github.com/joewilliams). It is now maintained, reviewed, and tested by Traffic Engineering team at GitHub. +OctoDNS was designed and authored by [Ross McFarland](https://github.com/ross) and [Joe Williams](https://github.com/joewilliams). It is now maintained, reviewed, and tested by Traffic Engineering team at GitHub. \ No newline at end of file diff --git a/octodns/provider/easydns.py b/octodns/provider/easydns.py new file mode 100644 index 0000000..bc58a83 --- /dev/null +++ b/octodns/provider/easydns.py @@ -0,0 +1,453 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict +from requests import Session +from time import sleep +import logging +import base64 + +from ..record import Record +from .base import BaseProvider + + +class EasyDNSClientException(Exception): + pass + + +class EasyDNSClientBadRequest(EasyDNSClientException): + + def __init__(self): + super(EasyDNSClientBadRequest, self).__init__('Bad request') + + +class EasyDNSClientNotFound(EasyDNSClientException): + + def __init__(self): + super(EasyDNSClientNotFound, self).__init__('Not Found') + + +class EasyDNSClientUnauthorized(EasyDNSClientException): + + def __init__(self): + super(EasyDNSClientUnauthorized, self).__init__('Unauthorized') + + +class EasyDNSClient(object): + # EasyDNS Sandbox API + SANDBOX = 'https://sandbox.rest.easydns.net' + # EasyDNS Live API + LIVE = 'https://rest.easydns.net' + # Default Currency CAD + defaultCurrency = 'CAD' + # Domain Portfolio + domainPortfolio = 'myport' + + def __init__(self, token, apikey, currency, portfolio, sandbox): + self.log = logging.getLogger('EasyDNSProvider[{}]'.format(id)) + self.token = token + self.apikey = apikey + self.defaultCurrency = currency + self.domainPortfolio = portfolio + self.apienv = 'sandbox' if sandbox else 'live' + authkey = '{}:{}'.format(self.token, self.apikey) + self.authkey = base64.b64encode(authkey.encode("utf-8")) + self.basepath = self.SANDBOX if sandbox else self.LIVE + sess = Session() + sess.headers.update({'Authorization': 'Basic {}'.format(self.authkey)}) + sess.headers.update({'accept': 'application/json'}) + self._sess = sess + + def _request(self, method, path, params=None, data=None): + url = '{}{}'.format(self.basepath, path) + resp = self._sess.request(method, url, params=params, json=data) + if resp.status_code == 400: + self.log.debug('Response code 400, path=%s', path) + if method == 'GET' and path[:8] == '/domain/': + raise EasyDNSClientNotFound() + raise EasyDNSClientBadRequest() + if resp.status_code == 401: + raise EasyDNSClientUnauthorized() + if resp.status_code == 403 or resp.status_code == 404: + raise EasyDNSClientNotFound() + resp.raise_for_status() + return resp + + def domain(self, name): + path = '/domain/{}'.format(name) + return self._request('GET', path).json() + + def domain_create(self, name): + # EasyDNS allows for new domains to be created for the purpose of DNS + # only, or with domain registration. This function creates a DNS only + # record expectig the domain to be registered already + path = '/domains/add/{}'.format(name) + domainData = {'service': 'dns', + 'term': 1, + 'dns_only': 1, + 'portfolio': self.domainPortfolio, + 'currency': self.defaultCurrency} + self._request('PUT', path, data=domainData).json() + + # EasyDNS creates default records for MX, A and CNAME for new domains, + # we need to delete those default record so we can sync with the source + # records, first we'll sleep for a second before gathering new records + # We also create default NS records, but they won't be deleted + sleep(1) + records = self.records(name, True) + for record in records: + if record['type'] in ('A', 'MX', 'CNAME'): + self.record_delete(name, record['id']) + + def records(self, zone_name, raw=False): + if raw: + path = '/zones/records/all/{}'.format(zone_name) + else: + path = '/zones/records/parsed/{}'.format(zone_name) + + ret = [] + resp = self._request('GET', path).json() + ret += resp['data'] + + # EasyDNS supports URL forwarding, stealth URL forwarding and DYNamic + # A records so we'll convert them to their underlying DNS record + # types before processing + for record in ret: + # change any apex record to empty string + if record['host'] == '@': + record['host'] = '' + + # change any apex value to zone name + if record['rdata'] == '@': + record['rdata'] = '{}.'.format(zone_name) + + # change "URL" & "STEALTH" to a "CNAME" + if record['type'] == "URL" or record['type'] == "STEALTH": + record['type'] = 'CNAME' + + if record['type'] == "DYN": + record['type'] = 'A' + + return ret + + def record_create(self, zone_name, params): + path = '/zones/records/add/{}/{}'.format(zone_name, params['type']) + # change empty name string to @, EasyDNS uses @ for apex record names + params['host'] = params['name'] + if params['host'] == '': + params['host'] = '@' + self._request('PUT', path, data=params) + + def record_delete(self, zone_name, record_id): + path = '/zones/records/{}/{}'.format(zone_name, record_id) + self._request('DELETE', path) + + +class EasyDNSProvider(BaseProvider): + ''' + EasyDNS provider using API v3 + + easydns: + class: octodns.provider.easydns.EasyDNSProvider + # Your EasyDNS API token (required) + token: foo + # Your EasyDNS API Key (required) + apikey: bar + # Use SandBox or Live environment, optional, defaults to live + sandbox: False + # Currency to use for creating domains, default CAD + defaultCurrency: CAD + # Domain Portfolio under which to create domains + portfolio: myport + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'TXT', + 'SRV', 'NAPTR')) + + def __init__(self, id, token, apikey, currency='CAD', portfolio='myport', + sandbox=False, *args, **kwargs): + self.log = logging.getLogger('EasyDNSProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, token=***', id) + super(EasyDNSProvider, self).__init__(id, *args, **kwargs) + self._client = EasyDNSClient(token, apikey, currency, portfolio, + sandbox) + self._zone_records = {} + + def _data_for_multiple(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['rdata'] for r in records] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + try: + flags, tag, value = record['rdata'].split(' ', 2) + except ValueError: + continue + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NAPTR(self, _type, records): + values = [] + for record in records: + try: + order, preference, flags, service, regexp, replacement = \ + record['rdata'].split(' ', 5) + except ValueError: + continue + values.append({ + 'flags': flags[1:-1], + 'order': order, + 'preference': preference, + 'regexp': regexp[1:-1], + 'replacement': replacement, + 'service': service[1:-1], + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def _data_for_CNAME(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': '{}'.format(record['rdata']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + values.append({ + 'preference': record['prio'], + 'exchange': '{}'.format(record['rdata']) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + data = '{}'.format(record['rdata']) + values.append(data) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_SRV(self, _type, records): + values = [] + record = records[0] + for record in records: + try: + priority, weight, port, target = record['rdata'].split(' ', 3) + except ValueError: + rdata = record['rdata'].split(' ', 3) + priority = 0 + weight = 0 + port = 0 + target = '' + if len(rdata) != 0 and rdata[0] != '': + priority = rdata[0] + if len(rdata) >= 2: + weight = rdata[1] + if len(rdata) >= 3: + port = rdata[2] + values.append({ + 'port': int(port), + 'priority': int(priority), + 'target': target, + 'weight': int(weight) + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = ['"' + value['rdata'].replace(';', '\\;') + + '"' for value in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = \ + self._client.records(zone.name[:-1]) + except EasyDNSClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + if _type not in self.SUPPORTS: + self.log.warning('populate: skipping unsupported %s record', + _type) + continue + values[record['host']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'rdata': value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + + def _params_for_CAA(self, record): + for value in record.values: + yield { + 'rdata': "{} {} {}".format(value.flags, value.tag, + value.value), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_NAPTR(self, record): + for value in record.values: + content = '{} {} "{}" "{}" "{}" {}'.format(value.order, + value.preference, + value.flags, + value.service, + value.regexp, + value.replacement) + yield { + 'rdata': content, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_single(self, record): + yield { + 'rdata': record.value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_CNAME = _params_for_single + + def _params_for_MX(self, record): + for value in record.values: + yield { + 'rdata': value.exchange, + 'name': record.name, + 'prio': value.preference, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_SRV(self, record): + for value in record.values: + yield { + 'rdata': "{} {} {} {}".format(value.priority, value.port, + value.weight, value.target), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type, + } + + def _params_for_TXT(self, record): + for value in record.values: + yield { + 'rdata': '"' + value.replace('\\;', ';') + '"', + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _apply_Create(self, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.record_create(new.zone.name[:-1], params) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + self.log.debug('apply_Delete: zone=%s, type=%s, host=%s', zone, + record['type'], record['host']) + if existing.name == record['host'] and \ + existing._type == record['type']: + self._client.record_delete(zone.name[:-1], record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + domain_name = desired.name[:-1] + try: + self._client.domain(domain_name) + except EasyDNSClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + self._client.domain_create(domain_name) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) diff --git a/tests/fixtures/easydns-records.json b/tests/fixtures/easydns-records.json new file mode 100644 index 0000000..ab16fc0 --- /dev/null +++ b/tests/fixtures/easydns-records.json @@ -0,0 +1,274 @@ +{ + "tm": 1000000000, + "data": [ + { + "id": "12340001", + "domain": "unit.tests", + "host": "@", + "ttl": "3600", + "prio": "0", + "type": "SOA", + "rdata": "dns1.easydns.com. zone.easydns.com. 2020010101 3600 600 604800 0", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340002", + "domain": "unit.tests", + "host": "@", + "ttl": "300", + "prio": "0", + "type": "A", + "rdata": "1.2.3.4", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340003", + "domain": "unit.tests", + "host": "@", + "ttl": "300", + "prio": "0", + "type": "DYN", + "rdata": "1.2.3.5", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340004", + "domain": "unit.tests", + "host": "@", + "ttl": "0", + "prio": null, + "type": "NS", + "rdata": "6.2.3.4.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340005", + "domain": "unit.tests", + "host": "@", + "ttl": "0", + "prio": null, + "type": "NS", + "rdata": "7.2.3.4.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340006", + "domain": "unit.tests", + "host": "@", + "ttl": "3600", + "prio": "0", + "type": "CAA", + "rdata": "0 issue ca.unit.tests", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340007", + "domain": "unit.tests", + "host": "_srv._tcp", + "ttl": "600", + "prio": "12", + "type": "SRV", + "rdata": "12 20 30 foo-2.unit.tests.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340008", + "domain": "unit.tests", + "host": "_srv._tcp", + "ttl": "600", + "prio": "12", + "type": "SRV", + "rdata": "10 20 30 foo-1.unit.tests.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340009", + "domain": "unit.tests", + "host": "aaaa", + "ttl": "600", + "prio": "0", + "type": "AAAA", + "rdata": "2601:644:500:e210:62f8:1dff:feb8:947a", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340010", + "domain": "unit.tests", + "host": "cname", + "ttl": "300", + "prio": null, + "type": "URL", + "rdata": "@", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340012", + "domain": "unit.tests", + "host": "mx", + "ttl": "300", + "prio": "10", + "type": "MX", + "rdata": "smtp-4.unit.tests.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340013", + "domain": "unit.tests", + "host": "mx", + "ttl": "300", + "prio": "20", + "type": "MX", + "rdata": "smtp-2.unit.tests.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340014", + "domain": "unit.tests", + "host": "mx", + "ttl": "300", + "prio": "30", + "type": "MX", + "rdata": "smtp-3.unit.tests.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340015", + "domain": "unit.tests", + "host": "mx", + "ttl": "300", + "prio": "40", + "type": "MX", + "rdata": "smtp-1.unit.tests.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340016", + "domain": "unit.tests", + "host": "naptr", + "ttl": "600", + "prio": null, + "type": "NAPTR", + "rdata": "100 100 'U' 'SIP+D2U' '!^.*$!sip:info@bar.example.com!' .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340017", + "domain": "unit.tests", + "host": "naptr", + "ttl": "600", + "prio": null, + "type": "NAPTR", + "rdata": "10 100 'S' 'SIP+D2U' '!^.*$!sip:info@bar.example.com!' .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340018", + "domain": "unit.tests", + "host": "sub", + "ttl": "3600", + "prio": null, + "type": "NS", + "rdata": "6.2.3.4.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340019", + "domain": "unit.tests", + "host": "sub", + "ttl": "0", + "prio": null, + "type": "NS", + "rdata": "7.2.3.4.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340020", + "domain": "unit.tests", + "host": "www", + "ttl": "300", + "prio": "0", + "type": "A", + "rdata": "2.2.3.6", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340021", + "domain": "unit.tests", + "host": "www.sub", + "ttl": "300", + "prio": "0", + "type": "A", + "rdata": "2.2.3.6", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340022", + "domain": "unit.tests", + "host": "included", + "ttl": "3600", + "prio": null, + "type": "CNAME", + "rdata": "unit.tests.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340011", + "domain": "unit.tests", + "host": "txt", + "ttl": "600", + "prio": "0", + "type": "TXT", + "rdata": "Bah bah black sheep", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340023", + "domain": "unit.tests", + "host": "txt", + "ttl": "600", + "prio": "0", + "type": "TXT", + "rdata": "have you any wool.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340024", + "domain": "unit.tests", + "host": "txt", + "ttl": "600", + "prio": "0", + "type": "TXT", + "rdata": "v=DKIM1;k=rsa;s=email;h=sha256;p=A\/kinda+of\/long\/string+with+numb3rs", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + } + ], + "count": 24, + "total": 24, + "start": 0, + "max": 1000, + "status": 200 +} \ No newline at end of file diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py new file mode 100644 index 0000000..6f39915 --- /dev/null +++ b/tests/test_octodns_provider_easydns.py @@ -0,0 +1,449 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import json +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.easydns import EasyDNSClientNotFound, \ + EasyDNSProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestEasyDNSProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + def test_populate(self): + provider = EasyDNSProvider('test', 'token', 'apikey') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"id":"unauthorized",' + '"message":"Unable to authenticate you."}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', text_type(ctx.exception)) + + # Bad request + with requests_mock() as mock: + mock.get(ANY, status_code=400, + text='{"id":"invalid",' + '"message":"Bad request"}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Bad request', text_type(ctx.exception)) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existent zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"id":"not_found","message":"The resource you ' + 'were accessing could not be found."}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://rest.easydns.net/zones/records/' + with open('tests/fixtures/easydns-records.json') as fh: + mock.get('{}{}'.format(base, 'parsed/unit.tests'), + text=fh.read()) + with open('tests/fixtures/easydns-records.json') as fh: + mock.get('{}{}'.format(base, 'all/unit.tests'), + text=fh.read()) + + provider.populate(zone) + self.assertEquals(13, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(13, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] + + def test_domain(self): + provider = EasyDNSProvider('test', 'token', 'apikey') + + with requests_mock() as mock: + base = 'https://rest.easydns.net/' + mock.get('{}{}'.format(base, 'domain/unit.tests'), status_code=400, + text='{"id":"not_found","message":"The resource you ' + 'were accessing could not be found."}') + + with self.assertRaises(Exception) as ctx: + provider._client.domain('unit.tests') + + self.assertEquals('Not Found', text_type(ctx.exception)) + + def test_apply_not_found(self): + provider = EasyDNSProvider('test', 'token', 'apikey') + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'test1', { + "name": "test1", + "ttl": 300, + "type": "A", + "value": "1.2.3.4", + })) + + with requests_mock() as mock: + base = 'https://rest.easydns.net/' + mock.get('{}{}'.format(base, 'domain/unit.tests'), status_code=404, + text='{"id":"not_found","message":"The resource you ' + 'were accessing could not be found."}') + mock.put('{}{}'.format(base, 'domains/add/unit.tests'), + status_code=200, + text='{"id":"OK","message":"Zone created."}') + mock.get('{}{}'.format(base, 'zones/records/parsed/unit.tests'), + status_code=404, + text='{"id":"not_found","message":"The resource you ' + 'were accessing could not be found."}') + mock.get('{}{}'.format(base, 'zones/records/all/unit.tests'), + status_code=404, + text='{"id":"not_found","message":"The resource you ' + 'were accessing could not be found."}') + + plan = provider.plan(wanted) + self.assertFalse(plan.exists) + self.assertEquals(1, len(plan.changes)) + with self.assertRaises(Exception) as ctx: + provider.apply(plan) + + self.assertEquals('Not Found', text_type(ctx.exception)) + + def test_domain_create(self): + provider = EasyDNSProvider('test', 'token', 'apikey') + domain_after_creation = { + "tm": 1000000000, + "data": [{ + "id": "12341001", + "domain": "unit.tests", + "host": "@", + "ttl": "0", + "prio": "0", + "type": "SOA", + "rdata": "dns1.easydns.com. zone.easydns.com. " + "2020010101 3600 600 604800 0", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, { + "id": "12341002", + "domain": "unit.tests", + "host": "@", + "ttl": "0", + "prio": "0", + "type": "NS", + "rdata": "LOCAL.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, { + "id": "12341003", + "domain": "unit.tests", + "host": "@", + "ttl": "0", + "prio": "0", + "type": "MX", + "rdata": "LOCAL.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }], + "count": 3, + "total": 3, + "start": 0, + "max": 1000, + "status": 200 + } + with requests_mock() as mock: + base = 'https://rest.easydns.net/' + mock.put('{}{}'.format(base, 'domains/add/unit.tests'), + status_code=201, text='{"id":"OK"}') + mock.get('{}{}'.format(base, 'zones/records/all/unit.tests'), + text=json.dumps(domain_after_creation)) + mock.delete(ANY, text='{"id":"OK"}') + provider._client.domain_create('unit.tests') + + def test_caa(self): + provider = EasyDNSProvider('test', 'token', 'apikey') + + # Invalid rdata records + caa_record_invalid = [{ + "domain": "unit.tests", + "host": "@", + "ttl": "3600", + "prio": "0", + "type": "CAA", + "rdata": "0", + }] + + # Valid rdata records + caa_record_valid = [{ + "domain": "unit.tests", + "host": "@", + "ttl": "3600", + "prio": "0", + "type": "CAA", + "rdata": "0 issue ca.unit.tests", + }] + + provider._data_for_CAA('CAA', caa_record_invalid) + provider._data_for_CAA('CAA', caa_record_valid) + + def test_naptr(self): + provider = EasyDNSProvider('test', 'token', 'apikey') + + # Invalid rdata records + naptr_record_invalid = [{ + "domain": "unit.tests", + "host": "naptr", + "ttl": "600", + "prio": "10", + "type": "NAPTR", + "rdata": "100", + }] + + # Valid rdata records + naptr_record_valid = [{ + "domain": "unit.tests", + "host": "naptr", + "ttl": "600", + "prio": "10", + "type": "NAPTR", + "rdata": "10 10 'U' 'SIP+D2U' '!^.*$!sip:info@bar.example.com!' .", + }] + + provider._data_for_NAPTR('NAPTR', naptr_record_invalid) + provider._data_for_NAPTR('NAPTR', naptr_record_valid) + + def test_srv(self): + provider = EasyDNSProvider('test', 'token', 'apikey') + + # Invalid rdata records + srv_invalid = [{ + "domain": "unit.tests", + "host": "_srv._tcp", + "ttl": "600", + "type": "SRV", + "rdata": "", + }] + srv_invalid2 = [{ + "domain": "unit.tests", + "host": "_srv._tcp", + "ttl": "600", + "type": "SRV", + "rdata": "11", + }] + srv_invalid3 = [{ + "domain": "unit.tests", + "host": "_srv._tcp", + "ttl": "600", + "type": "SRV", + "rdata": "12 30", + }] + srv_invalid4 = [{ + "domain": "unit.tests", + "host": "_srv._tcp", + "ttl": "600", + "type": "SRV", + "rdata": "13 40 1234", + }] + + # Valid rdata + srv_valid = [{ + "domain": "unit.tests", + "host": "_srv._tcp", + "ttl": "600", + "type": "SRV", + "rdata": "100 20 5678 foo-2.unit.tests.", + }] + + srv_invalid_content = provider._data_for_SRV('SRV', srv_invalid) + srv_invalid_content2 = provider._data_for_SRV('SRV', srv_invalid2) + srv_invalid_content3 = provider._data_for_SRV('SRV', srv_invalid3) + srv_invalid_content4 = provider._data_for_SRV('SRV', srv_invalid4) + srv_valid_content = provider._data_for_SRV('SRV', srv_valid) + + self.assertEqual(srv_valid_content['values'][0]['priority'], 100) + self.assertEqual(srv_invalid_content['values'][0]['priority'], 0) + self.assertEqual(srv_invalid_content2['values'][0]['priority'], 11) + self.assertEqual(srv_invalid_content3['values'][0]['priority'], 12) + self.assertEqual(srv_invalid_content4['values'][0]['priority'], 13) + + self.assertEqual(srv_valid_content['values'][0]['weight'], 20) + self.assertEqual(srv_invalid_content['values'][0]['weight'], 0) + self.assertEqual(srv_invalid_content2['values'][0]['weight'], 0) + self.assertEqual(srv_invalid_content3['values'][0]['weight'], 30) + self.assertEqual(srv_invalid_content4['values'][0]['weight'], 40) + + self.assertEqual(srv_valid_content['values'][0]['port'], 5678) + self.assertEqual(srv_invalid_content['values'][0]['port'], 0) + self.assertEqual(srv_invalid_content2['values'][0]['port'], 0) + self.assertEqual(srv_invalid_content3['values'][0]['port'], 0) + self.assertEqual(srv_invalid_content4['values'][0]['port'], 1234) + + self.assertEqual(srv_valid_content['values'][0]['target'], + 'foo-2.unit.tests.') + self.assertEqual(srv_invalid_content['values'][0]['target'], '') + self.assertEqual(srv_invalid_content2['values'][0]['target'], '') + self.assertEqual(srv_invalid_content3['values'][0]['target'], '') + self.assertEqual(srv_invalid_content4['values'][0]['target'], '') + + def test_apply(self): + provider = EasyDNSProvider('test', 'token', 'apikey') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + domain_after_creation = { + "tm": 1000000000, + "data": [{ + "id": "12341001", + "domain": "unit.tests", + "host": "@", + "ttl": "0", + "prio": "0", + "type": "SOA", + "rdata": "dns1.easydns.com. zone.easydns.com. 2020010101" + " 3600 600 604800 0", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, { + "id": "12341002", + "domain": "unit.tests", + "host": "@", + "ttl": "0", + "prio": "0", + "type": "NS", + "rdata": "LOCAL.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, { + "id": "12341003", + "domain": "unit.tests", + "host": "@", + "ttl": "0", + "prio": "0", + "type": "MX", + "rdata": "LOCAL.", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }], + "count": 3, + "total": 3, + "start": 0, + "max": 1000, + "status": 200 + } + + # non-existent domain, create everything + resp.json.side_effect = [ + EasyDNSClientNotFound, # no zone in populate + domain_after_creation + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 6 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + self.assertFalse(plan.exists) + + self.assertEquals(23, provider._client._request.call_count) + + provider._client._request.reset_mock() + + # delete 1 and update 1 + provider._client.records = Mock(return_value=[ + { + "id": "12342001", + "domain": "unit.tests", + "host": "www", + "ttl": "300", + "prio": "0", + "type": "A", + "rdata": "2.2.3.9", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, { + "id": "12342002", + "domain": "unit.tests", + "host": "www", + "ttl": "300", + "prio": "0", + "type": "A", + "rdata": "2.2.3.8", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, { + "id": "12342003", + "domain": "unit.tests", + "host": "test1", + "ttl": "3600", + "prio": "0", + "type": "A", + "rdata": "1.2.3.4", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'test1', { + "name": "test1", + "ttl": 300, + "type": "A", + "value": "1.2.3.4", + })) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + # recreate for update, and delete for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('PUT', '/zones/records/add/unit.tests/A', data={ + 'rdata': '1.2.3.4', + 'name': 'test1', + 'ttl': 300, + 'type': 'A', + 'host': 'test1', + }), + call('DELETE', '/zones/records/unit.tests/12342001'), + call('DELETE', '/zones/records/unit.tests/12342002'), + call('DELETE', '/zones/records/unit.tests/12342003') + ], any_order=True) From 317443b31b5dceea70e11b493a1cc13742251fbf Mon Sep 17 00:00:00 2001 From: John Dale Date: Thu, 9 Jul 2020 06:29:17 +0000 Subject: [PATCH 06/14] Corrected README.md Fix issue caused by making changes to the README.md through IDE which appears to have incorrectly encoded some of the lines that were not changed to add the EasyDNS provider details. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0cc2ac7..482ddc2 100644 --- a/README.md +++ b/README.md @@ -284,8 +284,8 @@ OctoDNS is licensed under the [MIT license](LICENSE). The MIT license grant is not for GitHub's trademarks, which include the logo designs. GitHub reserves all trademark and copyright rights in and to all GitHub trademarks. GitHub's logos include, for instance, the stylized designs that include "logo" in the file title in the following folder: https://github.com/github/octodns/tree/master/docs/logos/ -GitHub® and its stylized versions and the Invertocat mark are GitHub's Trademarks or registered Trademarks. When using GitHub's logos, be sure to follow the GitHub logo guidelines. +GitHub® and its stylized versions and the Invertocat mark are GitHub's Trademarks or registered Trademarks. When using GitHub's logos, be sure to follow the GitHub logo guidelines. ## Authors -OctoDNS was designed and authored by [Ross McFarland](https://github.com/ross) and [Joe Williams](https://github.com/joewilliams). It is now maintained, reviewed, and tested by Traffic Engineering team at GitHub. \ No newline at end of file +OctoDNS was designed and authored by [Ross McFarland](https://github.com/ross) and [Joe Williams](https://github.com/joewilliams). It is now maintained, reviewed, and tested by Traffic Engineering team at GitHub. From 4d006e94a212f2babce6aeeebaf3eaba3961e4d0 Mon Sep 17 00:00:00 2001 From: Phelps Williams Date: Wed, 15 Jul 2020 18:17:33 -0700 Subject: [PATCH 07/14] Adding environment variable record injection Per the discussion on https://github.com/github/octodns/issues/583 here is a work in progress of environment variable injection for discussion. --- octodns/source/envvar.py | 105 ++++++++++++++++++++++++++++ tests/test_octodns_source_envvar.py | 34 +++++++++ 2 files changed, 139 insertions(+) create mode 100644 octodns/source/envvar.py create mode 100644 tests/test_octodns_source_envvar.py diff --git a/octodns/source/envvar.py b/octodns/source/envvar.py new file mode 100644 index 0000000..b755632 --- /dev/null +++ b/octodns/source/envvar.py @@ -0,0 +1,105 @@ + +import logging +import os + +from ..record import Record +from .base import BaseSource + + +class EnvVarSourceException(Exception): + pass + + +class EnvironmentVariableNotFoundException(EnvVarSourceException): + def __init__(self, data): + super(EnvironmentVariableNotFoundException, self).__init__( + 'Unknown environment variable {}'.format(data)) + + +class EnvVarSource(BaseSource): + ''' + This source allows for environment variables to be embedded at octodns + execution time into zones. Intended to capture artifacts of deployment to + facilitate operational objectives. + + The TXT record generated will only have a single value. + + The record name cannot conflict with any other co-existing sources. If + this occurs, an exception will be thrown. + + Possible use cases include: + - Embedding a version number into a TXT record to monitor update + propagation across authoritative providers. + - Capturing identifying information about the deployment process to + record where and when the zone was updated. + + version: + class: octodns.source.envvar.EnvVarSource + # The environment variable in question, in this example the username + # currently executing octodns + variable: USER + # The TXT record name to embed the value found at the above + # environment variable + record: deployuser + # The TTL of the TXT record (optional, default 60) + ttl: 3600 + + This source is then combined with other sources in the octodns config + file: + + zones: + netflix.com.: + sources: + - yaml + - version + targets: + - ultra + - ns1 + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('TXT')) + + DEFAULT_TTL = 60 + + def __init__(self, id, variable, record, ttl=DEFAULT_TTL): + self.log = logging.getLogger('{}[{}]'.format( + self.__class__.__name__, id)) + self.log.debug('__init__: id=%s, variable=%s, record=%s, ' + 'ttl=%d', id, variable, record, ttl) + super(EnvVarSource, self).__init__(id) + self.envvar = variable + self.record = record + self.ttl = ttl + self.value = None + + def _read_variable(self): + self.value = os.environ.get(self.envvar) + if self.value is None: + raise EnvironmentVariableNotFoundException(self.envvar) + + self.log.debug('_read_variable: successfully loaded var=%s val=%s', + self.envvar, self.value) + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + # if target: + # TODO: Environment Variable Source cannot act as a target, + # throw exception? + # return + + before = len(zone.records) + + self._read_variable() + + # We don't need to worry about conflicting records here because the + # manager will deconflict sources on our behalf. + payload = {'ttl': self.ttl, 'type': 'TXT', 'values': [self.value]} + record = Record.new(zone, self.record, payload, source=self, + lenient=lenient) + zone.add_record(record, lenient=lenient) + + self.log.info('populate: found %s records, exists=False', + len(zone.records) - before) diff --git a/tests/test_octodns_source_envvar.py b/tests/test_octodns_source_envvar.py new file mode 100644 index 0000000..e562aa0 --- /dev/null +++ b/tests/test_octodns_source_envvar.py @@ -0,0 +1,34 @@ +from six import text_type +from unittest import TestCase +from unittest.mock import patch + +from octodns.source.envvar import EnvVarSource +from octodns.source.envvar import EnvironmentVariableNotFoundException +from octodns.zone import Zone + + +class TestEnvVarSource(TestCase): + + def test_read_variable(self): + envvar = 'OCTODNS_TEST_ENVIRONMENT_VARIABLE' + source = EnvVarSource('testid', envvar, 'recordname', ttl=120) + with self.assertRaises(EnvironmentVariableNotFoundException) as ctx: + source._read_variable() + msg = 'Unknown environment variable {}'.format(envvar) + self.assertEquals(msg, text_type(ctx.exception)) + + with patch.dict('os.environ', {envvar: 'testvalue'}): + source._read_variable() + self.assertEquals(source.value, 'testvalue') + + def test_populate(self): + envvar = 'TEST_VAR' + value = 'somevalue' + record = 'testrecord' + source = EnvVarSource('testid', envvar, record) + zone = Zone('unit.tests.', []) + + with patch.dict('os.environ', {envvar: value}): + source.populate(zone) + + # TODO: Validate zone and record From 0a342aa6c2586a87b8a0a7df74ce85e579a4aa96 Mon Sep 17 00:00:00 2001 From: Phelps Williams Date: Fri, 17 Jul 2020 12:09:20 -0700 Subject: [PATCH 08/14] EnvVar: Integrating review feedback and finishing tests --- octodns/source/envvar.py | 29 ++++++++++++----------------- tests/test_octodns_source_envvar.py | 19 +++++++++++++------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/octodns/source/envvar.py b/octodns/source/envvar.py index b755632..adf267a 100644 --- a/octodns/source/envvar.py +++ b/octodns/source/envvar.py @@ -40,7 +40,7 @@ class EnvVarSource(BaseSource): variable: USER # The TXT record name to embed the value found at the above # environment variable - record: deployuser + name: deployuser # The TTL of the TXT record (optional, default 60) ttl: 3600 @@ -62,42 +62,37 @@ class EnvVarSource(BaseSource): DEFAULT_TTL = 60 - def __init__(self, id, variable, record, ttl=DEFAULT_TTL): + def __init__(self, id, variable, name, ttl=DEFAULT_TTL): self.log = logging.getLogger('{}[{}]'.format( self.__class__.__name__, id)) - self.log.debug('__init__: id=%s, variable=%s, record=%s, ' - 'ttl=%d', id, variable, record, ttl) + self.log.debug('__init__: id=%s, variable=%s, name=%s, ' + 'ttl=%d', id, variable, name, ttl) super(EnvVarSource, self).__init__(id) self.envvar = variable - self.record = record + self.name = name self.ttl = ttl - self.value = None def _read_variable(self): - self.value = os.environ.get(self.envvar) - if self.value is None: + value = os.environ.get(self.envvar) + if value is None: raise EnvironmentVariableNotFoundException(self.envvar) self.log.debug('_read_variable: successfully loaded var=%s val=%s', - self.envvar, self.value) + self.envvar, value) + return value def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) - # if target: - # TODO: Environment Variable Source cannot act as a target, - # throw exception? - # return - before = len(zone.records) - self._read_variable() + value = self._read_variable() # We don't need to worry about conflicting records here because the # manager will deconflict sources on our behalf. - payload = {'ttl': self.ttl, 'type': 'TXT', 'values': [self.value]} - record = Record.new(zone, self.record, payload, source=self, + payload = {'ttl': self.ttl, 'type': 'TXT', 'values': [value]} + record = Record.new(zone, self.name, payload, source=self, lenient=lenient) zone.add_record(record, lenient=lenient) diff --git a/tests/test_octodns_source_envvar.py b/tests/test_octodns_source_envvar.py index e562aa0..0714883 100644 --- a/tests/test_octodns_source_envvar.py +++ b/tests/test_octodns_source_envvar.py @@ -18,17 +18,24 @@ class TestEnvVarSource(TestCase): self.assertEquals(msg, text_type(ctx.exception)) with patch.dict('os.environ', {envvar: 'testvalue'}): - source._read_variable() - self.assertEquals(source.value, 'testvalue') + value = source._read_variable() + self.assertEquals(value, 'testvalue') def test_populate(self): envvar = 'TEST_VAR' value = 'somevalue' - record = 'testrecord' - source = EnvVarSource('testid', envvar, record) - zone = Zone('unit.tests.', []) + name = 'testrecord' + zone_name = 'unit.tests.' + source = EnvVarSource('testid', envvar, name) + zone = Zone(zone_name, []) with patch.dict('os.environ', {envvar: value}): source.populate(zone) - # TODO: Validate zone and record + self.assertEquals(1, len(zone.records)) + record = list(zone.records)[0] + self.assertEquals(name, record.name) + self.assertEquals('{}.{}'.format(name, zone_name), record.fqdn) + self.assertEquals('TXT', record._type) + self.assertEquals(1, len(record.values)) + self.assertEquals(value, record.values[0]) From c75df0d8ed5a7fee2f7ff08d4987cfc390a870d6 Mon Sep 17 00:00:00 2001 From: Phelps Williams Date: Fri, 17 Jul 2020 12:29:17 -0700 Subject: [PATCH 09/14] Adding entry in readme for environment variable support --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d4e7171..995776a 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | | | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | +| [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | No CNAME support, missing `NA` geo target | From f4aa96abe55b33255548053bc6fd4188aba506df Mon Sep 17 00:00:00 2001 From: John Dale Date: Sat, 18 Jul 2020 08:03:27 +0000 Subject: [PATCH 10/14] Update to provide consistency * Replaced camelCase with snake_case * Change apikey to api_key * Added check on record name before delete during domain_create --- octodns/provider/easydns.py | 46 ++++++++++++++------------ tests/test_octodns_provider_easydns.py | 1 - 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/octodns/provider/easydns.py b/octodns/provider/easydns.py index bc58a83..c95ed5d 100644 --- a/octodns/provider/easydns.py +++ b/octodns/provider/easydns.py @@ -43,27 +43,28 @@ class EasyDNSClient(object): # EasyDNS Live API LIVE = 'https://rest.easydns.net' # Default Currency CAD - defaultCurrency = 'CAD' + default_currency = 'CAD' # Domain Portfolio - domainPortfolio = 'myport' + domain_portfolio = 'myport' - def __init__(self, token, apikey, currency, portfolio, sandbox): + def __init__(self, token, api_key, currency, portfolio, sandbox): self.log = logging.getLogger('EasyDNSProvider[{}]'.format(id)) self.token = token - self.apikey = apikey - self.defaultCurrency = currency - self.domainPortfolio = portfolio + self.api_key = api_key + self.default_currency = currency + self.domain_portfolio = portfolio self.apienv = 'sandbox' if sandbox else 'live' - authkey = '{}:{}'.format(self.token, self.apikey) - self.authkey = base64.b64encode(authkey.encode("utf-8")) - self.basepath = self.SANDBOX if sandbox else self.LIVE + auth_key = '{}:{}'.format(self.token, self.api_key) + self.auth_key = base64.b64encode(auth_key.encode("utf-8")) + self.base_path = self.SANDBOX if sandbox else self.LIVE sess = Session() - sess.headers.update({'Authorization': 'Basic {}'.format(self.authkey)}) + sess.headers.update({'Authorization': 'Basic {}' + .format(self.auth_key)}) sess.headers.update({'accept': 'application/json'}) self._sess = sess def _request(self, method, path, params=None, data=None): - url = '{}{}'.format(self.basepath, path) + url = '{}{}'.format(self.base_path, path) resp = self._sess.request(method, url, params=params, json=data) if resp.status_code == 400: self.log.debug('Response code 400, path=%s', path) @@ -86,12 +87,12 @@ class EasyDNSClient(object): # only, or with domain registration. This function creates a DNS only # record expectig the domain to be registered already path = '/domains/add/{}'.format(name) - domainData = {'service': 'dns', - 'term': 1, - 'dns_only': 1, - 'portfolio': self.domainPortfolio, - 'currency': self.defaultCurrency} - self._request('PUT', path, data=domainData).json() + domain_data = {'service': 'dns', + 'term': 1, + 'dns_only': 1, + 'portfolio': self.domain_portfolio, + 'currency': self.default_currency} + self._request('PUT', path, data=domain_data).json() # EasyDNS creates default records for MX, A and CNAME for new domains, # we need to delete those default record so we can sync with the source @@ -100,7 +101,8 @@ class EasyDNSClient(object): sleep(1) records = self.records(name, True) for record in records: - if record['type'] in ('A', 'MX', 'CNAME'): + if record['host'] in ('', 'www') \ + and record['type'] in ('A', 'MX', 'CNAME'): self.record_delete(name, record['id']) def records(self, zone_name, raw=False): @@ -156,11 +158,11 @@ class EasyDNSProvider(BaseProvider): # Your EasyDNS API token (required) token: foo # Your EasyDNS API Key (required) - apikey: bar + api_key: bar # Use SandBox or Live environment, optional, defaults to live sandbox: False # Currency to use for creating domains, default CAD - defaultCurrency: CAD + default_currency: CAD # Domain Portfolio under which to create domains portfolio: myport ''' @@ -169,12 +171,12 @@ class EasyDNSProvider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'TXT', 'SRV', 'NAPTR')) - def __init__(self, id, token, apikey, currency='CAD', portfolio='myport', + def __init__(self, id, token, api_key, currency='CAD', portfolio='myport', sandbox=False, *args, **kwargs): self.log = logging.getLogger('EasyDNSProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, token=***', id) super(EasyDNSProvider, self).__init__(id, *args, **kwargs) - self._client = EasyDNSClient(token, apikey, currency, portfolio, + self._client = EasyDNSClient(token, api_key, currency, portfolio, sandbox) self._zone_records = {} diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 6f39915..2681bf4 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -2,7 +2,6 @@ # # - from __future__ import absolute_import, division, print_function, \ unicode_literals From 427b8a1a061acb164edf0ecd1bafe497cce85f18 Mon Sep 17 00:00:00 2001 From: Justin B Newman Date: Mon, 20 Jul 2020 12:48:47 -0500 Subject: [PATCH 11/14] Add support for wildcard SRV records, as shown in RFC 2782 --- octodns/record/__init__.py | 2 +- tests/test_octodns_record.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 04eb2da..849e035 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1209,7 +1209,7 @@ class SrvValue(EqualityTupleMixin): class SrvRecord(_ValuesMixin, Record): _type = 'SRV' _value_type = SrvValue - _name_re = re.compile(r'^_[^\.]+\.[^\.]+') + _name_re = re.compile(r'^(\*|_[^\.]+)\.[^\.]+') @classmethod def validate(cls, name, fqdn, data): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index e2917b3..08a3e7a 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2155,6 +2155,18 @@ class TestRecordValidation(TestCase): } }) + # permit wildcard entries + Record.new(self.zone, '*._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'food.bar.baz.' + } + }) + # invalid name with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'neup', { From 5c248b476db1fd045c1fffdbfaa09086b2c0ddb4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 20 Jul 2020 13:23:40 -0700 Subject: [PATCH 12/14] According to docs ipaddress was 3.3, requires for ipaddress too Also corrects futures to 3.2 in requires --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c6cc97..dd1643f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,10 @@ dnspython==1.16.0 docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 -futures==3.2.0; python_version < '3.0' +futures==3.2.0; python_version < '3.2' google-cloud-core==1.3.0 google-cloud-dns==0.32.0 -ipaddress==1.0.23 +ipaddress==1.0.23; python_version < '3.3' jmespath==0.10.0 msrestazure==0.6.4 natsort==6.2.1 diff --git a/setup.py b/setup.py index 142b209..9394e7f 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', - 'ipaddress>=1.0.22; python_version<"3.2"', + 'ipaddress>=1.0.22; python_version<"3.3"', 'natsort>=5.5.0', 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2', From 9a2152d249a6c1d710671f893e637aebabaec081 Mon Sep 17 00:00:00 2001 From: John Dale Date: Mon, 20 Jul 2020 22:58:21 +0000 Subject: [PATCH 13/14] Update to remove URL and STEALTH records * Removed conversion of URL and STEALTH records to CNAME records * Updated test fixtures to remove URL conversion testing --- octodns/provider/easydns.py | 9 ++------- tests/fixtures/easydns-records.json | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/octodns/provider/easydns.py b/octodns/provider/easydns.py index c95ed5d..d2eb2da 100644 --- a/octodns/provider/easydns.py +++ b/octodns/provider/easydns.py @@ -115,9 +115,6 @@ class EasyDNSClient(object): resp = self._request('GET', path).json() ret += resp['data'] - # EasyDNS supports URL forwarding, stealth URL forwarding and DYNamic - # A records so we'll convert them to their underlying DNS record - # types before processing for record in ret: # change any apex record to empty string if record['host'] == '@': @@ -127,10 +124,8 @@ class EasyDNSClient(object): if record['rdata'] == '@': record['rdata'] = '{}.'.format(zone_name) - # change "URL" & "STEALTH" to a "CNAME" - if record['type'] == "URL" or record['type'] == "STEALTH": - record['type'] = 'CNAME' - + # EasyDNS supports DYNamic A records so we'll convert these + # to their underlying DNS record type before processing if record['type'] == "DYN": record['type'] = 'A' diff --git a/tests/fixtures/easydns-records.json b/tests/fixtures/easydns-records.json index ab16fc0..9d0f5e1 100644 --- a/tests/fixtures/easydns-records.json +++ b/tests/fixtures/easydns-records.json @@ -106,7 +106,7 @@ "host": "cname", "ttl": "300", "prio": null, - "type": "URL", + "type": "CNAME", "rdata": "@", "geozone_id": "0", "last_mod": "2020-01-01 01:01:01" @@ -271,4 +271,4 @@ "start": 0, "max": 1000, "status": 200 -} \ No newline at end of file +} From 9e990632c4d2f1ea6e1ce48688a43703b8bc37e1 Mon Sep 17 00:00:00 2001 From: John Dale Date: Tue, 21 Jul 2020 19:58:24 +0000 Subject: [PATCH 14/14] Update to remove DYNamic A records * Removed conversion of DYN records to A records * Updated test fixtures to change test DYN to an A record --- octodns/provider/easydns.py | 5 ----- tests/fixtures/easydns-records.json | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/octodns/provider/easydns.py b/octodns/provider/easydns.py index d2eb2da..835fcb9 100644 --- a/octodns/provider/easydns.py +++ b/octodns/provider/easydns.py @@ -124,11 +124,6 @@ class EasyDNSClient(object): if record['rdata'] == '@': record['rdata'] = '{}.'.format(zone_name) - # EasyDNS supports DYNamic A records so we'll convert these - # to their underlying DNS record type before processing - if record['type'] == "DYN": - record['type'] = 'A' - return ret def record_create(self, zone_name, params): diff --git a/tests/fixtures/easydns-records.json b/tests/fixtures/easydns-records.json index 9d0f5e1..c3718b5 100644 --- a/tests/fixtures/easydns-records.json +++ b/tests/fixtures/easydns-records.json @@ -29,7 +29,7 @@ "host": "@", "ttl": "300", "prio": "0", - "type": "DYN", + "type": "A", "rdata": "1.2.3.5", "geozone_id": "0", "last_mod": "2020-01-01 01:01:01"